01HHAFHSSABNADJEE79MAHVNHB: test-repair

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HHAFK8NHE8260CPCN14RS312

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12023-12-10T18:46:38.187Zjob dependencies complete; ready to run (waiting for 21 m 42 s)
22023-12-10T18:47:58.126Zjob assigned to worker 01HHAGVJMBH2NZKAYB9DF5H3VR (queued for 1 m 19 s)
32023-12-10T18:47:59.017Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42023-12-10T18:48:09.786Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52023-12-10T18:48:09.786Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62023-12-10T18:48:34.781Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72023-12-10T18:48:34.782Zdownloading input: /input/build/work/bins/crutest.gz
82023-12-10T18:49:06.014Zdownloaded input: /input/build/work/bins/crutest.gz
92023-12-10T18:49:06.014Zdownloading input: /input/build/work/bins/dsc.gz
102023-12-10T18:49:25.857Zdownloaded input: /input/build/work/bins/dsc.gz
112023-12-10T18:49:25.867Zdownloading input: /input/build/work/scripts/perf-downstairs-tick.d
122023-12-10T18:49:25.881Zdownloaded input: /input/build/work/scripts/perf-downstairs-tick.d
132023-12-10T18:49:25.881Zdownloading input: /input/build/work/scripts/test_ds.sh
142023-12-10T18:49:25.900Zdownloaded input: /input/build/work/scripts/test_ds.sh
152023-12-10T18:49:25.900Zdownloading input: /input/build/work/scripts/test_live_repair.sh
162023-12-10T18:49:25.923Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
172023-12-10T18:49:25.923Zdownloading input: /input/build/work/scripts/test_repair.sh
182023-12-10T18:49:25.939Zdownloaded input: /input/build/work/scripts/test_repair.sh
192023-12-10T18:49:25.939Zdownloading input: /input/build/work/scripts/test_replay.sh
202023-12-10T18:49:26.371Zdownloaded input: /input/build/work/scripts/test_replay.sh
212023-12-10T18:49:26.371Zdownloading input: /input/build/work/scripts/test_up.sh
222023-12-10T18:49:26.505Zdownloaded input: /input/build/work/scripts/test_up.sh
232023-12-10T18:49:26.505Zdownloading input: /input/build/work/scripts/upstairs_info.d
242023-12-10T18:49:26.526Zdownloaded input: /input/build/work/scripts/upstairs_info.d
252023-12-10T18:49:26.538Zdownloading input: /input/build/tmp/cargo-test-out.log
262023-12-10T18:49:29.367Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
272023-12-10T18:49:29.367Zstarting task 0: "setup"
282023-12-10T18:49:29.495Z++ uname -s
292023-12-10T18:49:29.504Z+ kern=SunOS
302023-12-10T18:49:29.504Z+ case "$kern" in
312023-12-10T18:49:29.504Z+ groupadd -g 12345 build
322023-12-10T18:49:29.504Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
332023-12-10T18:49:31.520Z+ zfs create -o mountpoint=/work rpool/work
342023-12-10T18:49:32.110Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
352023-12-10T18:49:32.114Z+ home_fs=zfs
362023-12-10T18:49:32.114Z+ [[ zfs == autofs ]]
372023-12-10T18:49:32.114Z+ mkdir -p /home/build
382023-12-10T18:49:32.114Z+ chown build:build /home/build /work
392023-12-10T18:49:33.757Z+ chmod 0700 /home/build /work
402023-12-10T18:49:34.245Zprocess exited: duration 4356 ms, exit code 0
 
412023-12-10T18:49:34.712Zstarting task 1: "authentication"
422023-12-10T18:49:35.308Zprocess exited: duration 598 ms, exit code 0
 
432023-12-10T18:49:35.345Zstarting task 2: "build"
442023-12-10T18:49:35.390Z+ banner cores
452023-12-10T18:49:35.415Z
462023-12-10T18:49:35.416Z #### #### ##### ###### ####
472023-12-10T18:49:35.416Z # # # # # # # #
482023-12-10T18:49:35.416Z # # # # # ##### ####
492023-12-10T18:49:35.416Z # # # ##### # #
502023-12-10T18:49:35.417Z # # # # # # # # #
512023-12-10T18:49:35.417Z #### #### # # ###### ####
522023-12-10T18:49:35.417Z
532023-12-10T18:49:35.428Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
542023-12-10T18:49:35.437Z+ echo 'input bins dir contains:'
552023-12-10T18:49:35.437Zinput bins dir contains:
562023-12-10T18:49:35.438Z+ ls -ltr /input/build/work/bins
572023-12-10T18:49:35.438Ztotal 707387
582023-12-10T18:49:35.438Z-rw-r--r-- 1 root root 110051390 Dec 10 18:48 crucible-downstairs.gz
592023-12-10T18:49:35.439Z-rw-r--r-- 1 root root 86835243 Dec 10 18:48 crucible-hammer.gz
602023-12-10T18:49:35.439Z-rw-r--r-- 1 root root 95895548 Dec 10 18:49 crutest.gz
612023-12-10T18:49:35.439Z-rw-r--r-- 1 root root 69165492 Dec 10 18:49 dsc.gz
622023-12-10T18:49:35.439Z+ echo 'input script dir contains:'
632023-12-10T18:49:35.439Zinput script dir contains:
642023-12-10T18:49:35.440Z+ ls -ltr /input/build/work/scripts
652023-12-10T18:49:35.441Ztotal 38
662023-12-10T18:49:35.441Z-rw-r--r-- 1 root root 3056 Dec 10 18:49 perf-downstairs-tick.d
672023-12-10T18:49:35.441Z-rw-r--r-- 1 root root 939 Dec 10 18:49 test_ds.sh
682023-12-10T18:49:35.441Z-rw-r--r-- 1 root root 3746 Dec 10 18:49 test_live_repair.sh
692023-12-10T18:49:35.441Z-rw-r--r-- 1 root root 6416 Dec 10 18:49 test_repair.sh
702023-12-10T18:49:35.441Z-rw-r--r-- 1 root root 2969 Dec 10 18:49 test_replay.sh
712023-12-10T18:49:35.442Z-rw-r--r-- 1 root root 9465 Dec 10 18:49 test_up.sh
722023-12-10T18:49:35.442Z-rw-r--r-- 1 root root 2796 Dec 10 18:49 upstairs_info.d
732023-12-10T18:49:35.442Z+ pfexec chmod +x /input/build/work/scripts/perf-downstairs-tick.d /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh /input/build/work/scripts/upstairs_info.d
742023-12-10T18:49:35.451Z+ echo ' chmod input script dir contains:'
752023-12-10T18:49:35.452Z chmod input script dir contains:
762023-12-10T18:49:35.452Z+ ls -ltr /input/build/work/scripts
772023-12-10T18:49:35.453Ztotal 38
782023-12-10T18:49:35.453Z-rwxr-xr-x 1 root root 3056 Dec 10 18:49 perf-downstairs-tick.d
792023-12-10T18:49:35.453Z-rwxr-xr-x 1 root root 939 Dec 10 18:49 test_ds.sh
802023-12-10T18:49:35.453Z-rwxr-xr-x 1 root root 3746 Dec 10 18:49 test_live_repair.sh
812023-12-10T18:49:35.454Z-rwxr-xr-x 1 root root 6416 Dec 10 18:49 test_repair.sh
822023-12-10T18:49:35.454Z-rwxr-xr-x 1 root root 2969 Dec 10 18:49 test_replay.sh
832023-12-10T18:49:35.454Z-rwxr-xr-x 1 root root 9465 Dec 10 18:49 test_up.sh
842023-12-10T18:49:35.455Z-rwxr-xr-x 1 root root 2796 Dec 10 18:49 upstairs_info.d
852023-12-10T18:49:35.455Z+ banner unpack
862023-12-10T18:49:35.455Z
872023-12-10T18:49:35.455Z # # # # ##### ## #### # #
882023-12-10T18:49:35.456Z # # ## # # # # # # # # #
892023-12-10T18:49:35.456Z # # # # # # # # # # ####
902023-12-10T18:49:35.456Z # # # # # ##### ###### # # #
912023-12-10T18:49:35.456Z # # # ## # # # # # # #
922023-12-10T18:49:35.457Z #### # # # # # #### # #
932023-12-10T18:49:35.457Z
942023-12-10T18:49:35.457Z+ mkdir -p /var/tmp/bins
952023-12-10T18:49:35.457Z+ for t in "$input/bins/"*.gz
962023-12-10T18:49:35.458Z++ basename /input/build/work/bins/crucible-downstairs.gz
972023-12-10T18:49:35.458Z+ b=crucible-downstairs.gz
982023-12-10T18:49:35.458Z+ b=crucible-downstairs
992023-12-10T18:49:35.458Z+ gunzip
1002023-12-10T18:49:39.341Z+ chmod +x /var/tmp/bins/crucible-downstairs
1012023-12-10T18:49:39.376Z+ for t in "$input/bins/"*.gz
1022023-12-10T18:49:39.377Z++ basename /input/build/work/bins/crucible-hammer.gz
1032023-12-10T18:49:39.377Z+ b=crucible-hammer.gz
1042023-12-10T18:49:39.378Z+ b=crucible-hammer
1052023-12-10T18:49:39.378Z+ gunzip
1062023-12-10T18:49:42.375Z+ chmod +x /var/tmp/bins/crucible-hammer
1072023-12-10T18:49:42.386Z+ for t in "$input/bins/"*.gz
1082023-12-10T18:49:42.387Z++ basename /input/build/work/bins/crutest.gz
1092023-12-10T18:49:42.387Z+ b=crutest.gz
1102023-12-10T18:49:42.387Z+ b=crutest
1112023-12-10T18:49:42.388Z+ gunzip
1122023-12-10T18:49:45.609Z+ chmod +x /var/tmp/bins/crutest
1132023-12-10T18:49:45.636Z+ for t in "$input/bins/"*.gz
1142023-12-10T18:49:45.637Z++ basename /input/build/work/bins/dsc.gz
1152023-12-10T18:49:45.637Z+ b=dsc.gz
1162023-12-10T18:49:45.638Z+ b=dsc
1172023-12-10T18:49:45.638Z+ gunzip
1182023-12-10T18:49:48.075Z+ chmod +x /var/tmp/bins/dsc
1192023-12-10T18:49:48.088Z+ export BINDIR=/var/tmp/bins
1202023-12-10T18:49:48.088Z+ BINDIR=/var/tmp/bins
1212023-12-10T18:49:48.088Z+ banner repair
1222023-12-10T18:49:48.088Z
1232023-12-10T18:49:48.088Z ##### ###### ##### ## # #####
1242023-12-10T18:49:48.088Z # # # # # # # # # #
1252023-12-10T18:49:48.088Z # # ##### # # # # # # #
1262023-12-10T18:49:48.089Z ##### # ##### ###### # #####
1272023-12-10T18:49:48.089Z # # # # # # # # #
1282023-12-10T18:49:48.089Z # # ###### # # # # # #
1292023-12-10T18:49:48.090Z
1302023-12-10T18:49:48.090Z+ ptime -m bash /input/build/work/scripts/test_repair.sh -N
1312023-12-10T18:49:48.123ZTurn off color for downstairs dump
1322023-12-10T18:49:48.133ZCreating region directory at: /var/tmp/test_repair
1332023-12-10T18:49:48.133ZCreating dsc directory at: /tmp/dsc
1342023-12-10T18:49:48.429ZDownstairs region 0 created at /var/tmp/test_repair/8810 in 0.31158254
1352023-12-10T18:49:48.510ZDownstairs region 1 created at /var/tmp/test_repair/8820 in 0.077810675
1362023-12-10T18:49:48.659ZDownstairs region 2 created at /var/tmp/test_repair/8830 in 0.061274663
1372023-12-10T18:49:48.659ZRegion set with 3 regions was created
1382023-12-10T18:49:48.676Z/var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1161 1162 1163
1392023-12-10T18:49:48.676ZDump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
1402023-12-10T18:49:48.676Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.431441628Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165}
1412023-12-10T18:49:48.676Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.432476955Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165}
1422023-12-10T18:49:48.676Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.432530436Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165}
1432023-12-10T18:49:48.676Z{"msg":"Crucible 094b3185-e5ad-47fd-8f58-866bd75278a7 has session id: fe988fab-1218-46fa-9b9a-abe601512ec8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.4325542Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1442023-12-10T18:49:48.676Z{"msg":"Upstairs opts: Upstairs UUID: 094b3185-e5ad-47fd-8f58-866bd75278a7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.432714659Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1452023-12-10T18:49:48.676Z{"msg":"Crucible stats registered with UUID: 094b3185-e5ad-47fd-8f58-866bd75278a7","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.432767977Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1462023-12-10T18:49:48.677ZCrucible runtime is spawned
1472023-12-10T18:49:48.677Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.434442586Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8","task":"up_listen"}
1482023-12-10T18:49:48.677Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.434534122Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1492023-12-10T18:49:48.678Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.434574677Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1502023-12-10T18:49:48.678ZThe guest has requested activation with gen:1
1512023-12-10T18:49:48.678Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.434741248Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"looper":"1","session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1522023-12-10T18:49:48.678Z{"msg":"[1] 094b3185-e5ad-47fd-8f58-866bd75278a7 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.435599654Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"looper":"1","session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1532023-12-10T18:49:48.678Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.43565803Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"looper":"0","session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1542023-12-10T18:49:48.678Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.435840275Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"looper":"2","session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1552023-12-10T18:49:48.678Z{"msg":"[0] 094b3185-e5ad-47fd-8f58-866bd75278a7 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.435934607Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"looper":"0","session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1562023-12-10T18:49:48.678Z{"msg":"[2] 094b3185-e5ad-47fd-8f58-866bd75278a7 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.435978248Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"looper":"2","session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1572023-12-10T18:49:48.678Z{"msg":"094b3185-e5ad-47fd-8f58-866bd75278a7 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.43663358Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1582023-12-10T18:49:48.678Z{"msg":"Set desired generation to :1","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.436808157Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1592023-12-10T18:49:48.678Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.436986256Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1602023-12-10T18:49:48.678Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.438057548Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1612023-12-10T18:49:48.679Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.438201505Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1622023-12-10T18:49:48.679Z{"msg":"[1] 094b3185-e5ad-47fd-8f58-866bd75278a7 (fe988fab-1218-46fa-9b9a-abe601512ec8) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.439565992Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1632023-12-10T18:49:48.679Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.43962132Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1642023-12-10T18:49:48.679Z{"msg":"[1] client is_active_req TRUE, promote! session fe988fab-1218-46fa-9b9a-abe601512ec8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.439665285Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1652023-12-10T18:49:48.825Z{"msg":"[0] 094b3185-e5ad-47fd-8f58-866bd75278a7 (fe988fab-1218-46fa-9b9a-abe601512ec8) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.439704696Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1662023-12-10T18:49:48.826Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.439770887Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1672023-12-10T18:49:48.826Z{"msg":"[2] 094b3185-e5ad-47fd-8f58-866bd75278a7 (fe988fab-1218-46fa-9b9a-abe601512ec8) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.439815619Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1682023-12-10T18:49:48.826Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.439850728Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1692023-12-10T18:49:48.838Z{"msg":"[0] client is_active_req TRUE, promote! session fe988fab-1218-46fa-9b9a-abe601512ec8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.439913219Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1702023-12-10T18:49:48.839Z{"msg":"[2] client is_active_req TRUE, promote! session fe988fab-1218-46fa-9b9a-abe601512ec8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.439955308Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1712023-12-10T18:49:48.839Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.443462847Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1722023-12-10T18:49:48.839Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.443544149Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1732023-12-10T18:49:48.839Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.443583801Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1742023-12-10T18:49:48.839Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.443631167Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1752023-12-10T18:49:48.839Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.443710912Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1762023-12-10T18:49:48.839Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.443761916Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1772023-12-10T18:49:48.840Z{"msg":"094b3185-e5ad-47fd-8f58-866bd75278a7 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.44383583Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1782023-12-10T18:49:48.840Z{"msg":"094b3185-e5ad-47fd-8f58-866bd75278a7 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.443910004Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1792023-12-10T18:49:48.840Z{"msg":"094b3185-e5ad-47fd-8f58-866bd75278a7 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.444050223Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1802023-12-10T18:49:48.840Z{"msg":"[0] 094b3185-e5ad-47fd-8f58-866bd75278a7 (fe988fab-1218-46fa-9b9a-abe601512ec8) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.444471401Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1812023-12-10T18:49:48.840Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.444581411Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1822023-12-10T18:49:48.840Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:49:47.444628353Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1832023-12-10T18:49:48.841Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.444669981Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1842023-12-10T18:49:48.841Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.44471367Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1852023-12-10T18:49:48.841Z{"msg":"094b3185-e5ad-47fd-8f58-866bd75278a7 WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.444787145Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1862023-12-10T18:49:48.841Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.44483125Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1872023-12-10T18:49:48.841Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.444899983Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1882023-12-10T18:49:48.843Z{"msg":"[2] 094b3185-e5ad-47fd-8f58-866bd75278a7 (fe988fab-1218-46fa-9b9a-abe601512ec8) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.445012751Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1892023-12-10T18:49:48.844Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.445077594Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1902023-12-10T18:49:48.844Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:49:47.445103228Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1912023-12-10T18:49:48.844Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.445145103Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1922023-12-10T18:49:48.844Z{"msg":"[1] 094b3185-e5ad-47fd-8f58-866bd75278a7 (fe988fab-1218-46fa-9b9a-abe601512ec8) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.445170589Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1932023-12-10T18:49:48.844Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.44521188Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1942023-12-10T18:49:48.844Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:49:47.44523483Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1952023-12-10T18:49:48.845Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.445276315Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1962023-12-10T18:49:48.845Z{"msg":"094b3185-e5ad-47fd-8f58-866bd75278a7 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.445300128Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1972023-12-10T18:49:48.845Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.445320375Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1982023-12-10T18:49:48.848Z{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.445341505Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
1992023-12-10T18:49:48.848Z{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.445382195Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2002023-12-10T18:49:48.849Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.445406335Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2012023-12-10T18:49:48.849Z{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.445429925Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2022023-12-10T18:49:48.849Z{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.44545117Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2032023-12-10T18:49:48.849Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.44549336Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2042023-12-10T18:49:48.849Z{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.445516582Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2052023-12-10T18:49:48.849Z{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.44555795Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2062023-12-10T18:49:48.850Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.445581712Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2072023-12-10T18:49:48.850Z{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.445625652Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2082023-12-10T18:49:48.850Z{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.445648707Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2092023-12-10T18:49:48.850Z{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.445668283Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2102023-12-10T18:49:48.850Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.446216932Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2112023-12-10T18:49:48.851Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.446298592Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2122023-12-10T18:49:48.851Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.446322392Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2132023-12-10T18:49:48.851Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.446342648Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2142023-12-10T18:49:48.851Z{"msg":"094b3185-e5ad-47fd-8f58-866bd75278a7 is now active with session: fe988fab-1218-46fa-9b9a-abe601512ec8","v":0,"name":"crucible","level":The guest has finished waiting for activation with:1
2152023-12-10T18:49:48.851Z30Wait for a query_work_queue command to finish before sending IO
2162023-12-10T18:49:48.851Z,"time":"2023-12-10T18:49:47.446476651Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2172023-12-10T18:49:48.852Z{"msg":"094b3185-e5ad-47fd-8f58-866bd75278a7 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.446670776Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2182023-12-10T18:49:48.852Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.446717793Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2192023-12-10T18:49:48.852Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.446758553Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2202023-12-10T18:49:48.852Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.446837813Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2212023-12-10T18:49:48.852Z{"msg":"094b3185-e5ad-47fd-8f58-866bd75278a7 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.446866179Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2222023-12-10T18:49:48.852Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.446888077Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2232023-12-10T18:49:48.852Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.44690837Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2242023-12-10T18:49:48.853Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.446928348Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2252023-12-10T18:49:48.853Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.44696992Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2262023-12-10T18:49:48.853Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.446992388Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2272023-12-10T18:49:48.853Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.447012125Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2282023-12-10T18:49:48.853Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.447058461Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2292023-12-10T18:49:48.853Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.447083831Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2302023-12-10T18:49:48.854Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.447103959Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2312023-12-10T18:49:48.854Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:47.44712394Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2322023-12-10T18:49:48.855ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2332023-12-10T18:49:48.855ZFill test
2342023-12-10T18:49:48.882ZRead and Verify all blocks (0..600 range:false)
2352023-12-10T18:49:49.026ZWrote out file "/tmp/test_repair_verify.data"
2362023-12-10T18:49:49.026ZCLIENT: Tests done. All submitted work has been ACK'd
2372023-12-10T18:49:49.053Z----------------------------------------------------------------
2382023-12-10T18:49:49.053Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:6
2392023-12-10T18:49:49.053ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
2402023-12-10T18:49:49.053Z 8 Acked 1007 Read 100 Done Done Done false
2412023-12-10T18:49:49.053Z 9 Acked 1008 Read 100 Done Done Done false
2422023-12-10T18:49:49.054Z 10 Acked 1009 Read 100 Done Done Done false
2432023-12-10T18:49:49.054Z 11 Acked 1010 Read 100 Done Done Done false
2442023-12-10T18:49:49.054Z 12 Acked 1011 Read 100 Done Done Done false
2452023-12-10T18:49:49.054Z 13 Acked 1012 Read 100 Done Sent Done false
2462023-12-10T18:49:49.054Z STATES DS:0 DS:1 DS:2 TOTAL
2472023-12-10T18:49:49.054Z New 0 0 0 0
2482023-12-10T18:49:49.055Z Sent 0 1 0 1
2492023-12-10T18:49:49.055Z Done 6 5 6 17
2502023-12-10T18:49:49.055Z Skipped 0 0 0 0
2512023-12-10T18:49:49.055Z Error 0 0 0 0
2522023-12-10T18:49:49.055ZLast Flush: 1006 1006 1006
2532023-12-10T18:49:49.055ZDownstairs last five completed: 1006 1005 1004 1003 1002 1001
2542023-12-10T18:49:49.055ZUpstairs last five completed: 13 12 11 10 9 8
2552023-12-10T18:49:49.055ZCLIENT: Up:0 ds:6 act:3
2562023-12-10T18:49:49.682Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:48.447277239Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2572023-12-10T18:49:49.682Z{"msg":"No Live Repair required at this time","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:48.447367974Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1165,"session_id":"fe988fab-1218-46fa-9b9a-abe601512ec8"}
2582023-12-10T18:49:52.971Z----------------------------------------------------------------
2592023-12-10T18:49:52.971Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0
2602023-12-10T18:49:52.972ZDownstairs last five completed: 1013 1012 1011 1010 1009 1008
2612023-12-10T18:49:52.972ZUpstairs last five completed: 14 13 12 11 10 9
2622023-12-10T18:49:52.972ZCLIENT: Up:0 ds:0 act:3
2632023-12-10T18:49:52.973ZCLIENT: All crucible jobs finished, exiting program
2642023-12-10T18:49:52.997Z
2652023-12-10T18:49:52.998ZBegin loop 1
2662023-12-10T18:49:52.998ZDownstairs to restart: 0
2672023-12-10T18:49:53.450Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.778771987Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167}
2682023-12-10T18:49:53.450Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.779193597Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167}
2692023-12-10T18:49:53.450Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.779262682Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167}
2702023-12-10T18:49:53.451Z{"msg":"Crucible e68e8be9-3bec-4457-8eba-a77c2680a871 has session id: 8654b076-ab9a-4a46-b323-754eaa738ae3","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.779290107Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
2712023-12-10T18:49:53.451Z{"msg":"Upstairs opts: Upstairs UUID: e68e8be9-3bec-4457-8eba-a77c2680a871, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.779371247Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
2722023-12-10T18:49:53.451ZCrucible runtime is spawned
2732023-12-10T18:49:53.451Z{"msg":"Crucible stats registered with UUID: e68e8be9-3bec-4457-8eba-a77c2680a871","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.779427961Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
2742023-12-10T18:49:53.451ZThe guest has requested activation with gen:2
2752023-12-10T18:49:53.451Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.77964276Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"looper":"0","session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
2762023-12-10T18:49:53.451Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.779766362Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3","task":"up_listen"}
2772023-12-10T18:49:53.451Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.77985221Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
2782023-12-10T18:49:53.452Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.779892853Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
2792023-12-10T18:49:53.453Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.779947224Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"looper":"1","session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
2802023-12-10T18:49:53.488Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.780084182Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"looper":"2","session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
2812023-12-10T18:49:53.488Z{"msg":"[1] e68e8be9-3bec-4457-8eba-a77c2680a871 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.780190854Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"looper":"1","session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
2822023-12-10T18:49:53.488Z{"msg":"[2] e68e8be9-3bec-4457-8eba-a77c2680a871 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.780318138Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"looper":"2","session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
2832023-12-10T18:49:53.488Z{"msg":"e68e8be9-3bec-4457-8eba-a77c2680a871 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.780657372Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
2842023-12-10T18:49:53.489Z{"msg":"Set desired generation to :2","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.780719981Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
2852023-12-10T18:49:53.489Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.780888244Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
2862023-12-10T18:49:53.489Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.781293126Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
2872023-12-10T18:49:53.489Z{"msg":"[1] e68e8be9-3bec-4457-8eba-a77c2680a871 (8654b076-ab9a-4a46-b323-754eaa738ae3) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.781648999Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
2882023-12-10T18:49:53.489Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.781711695Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
2892023-12-10T18:49:53.489Z{"msg":"[1] client is_active_req TRUE, promote! session 8654b076-ab9a-4a46-b323-754eaa738ae3","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.781736662Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
2902023-12-10T18:49:53.489Z{"msg":"[2] e68e8be9-3bec-4457-8eba-a77c2680a871 (8654b076-ab9a-4a46-b323-754eaa738ae3) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.781875637Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
2912023-12-10T18:49:53.489Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.781941226Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
2922023-12-10T18:49:53.490Z{"msg":"[2] client is_active_req TRUE, promote! session 8654b076-ab9a-4a46-b323-754eaa738ae3","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.782040576Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
2932023-12-10T18:49:53.490Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.782350576Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
2942023-12-10T18:49:53.490Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.782472564Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
2952023-12-10T18:49:53.490Z{"msg":"e68e8be9-3bec-4457-8eba-a77c2680a871 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.782550046Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
2962023-12-10T18:49:53.490Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.782691081Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
2972023-12-10T18:49:53.490Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.7827374Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
2982023-12-10T18:49:53.490Z{"msg":"e68e8be9-3bec-4457-8eba-a77c2680a871 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.782788488Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
2992023-12-10T18:49:53.490Z{"msg":"[1] e68e8be9-3bec-4457-8eba-a77c2680a871 (8654b076-ab9a-4a46-b323-754eaa738ae3) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.78322838Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3002023-12-10T18:49:53.491Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.783298463Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3012023-12-10T18:49:53.491Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:49:51.783338857Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3022023-12-10T18:49:53.491Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.783397813Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3032023-12-10T18:49:53.491Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.78342319Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3042023-12-10T18:49:53.491Z{"msg":"e68e8be9-3bec-4457-8eba-a77c2680a871 New WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.783465731Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3052023-12-10T18:49:53.491Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.783488028Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3062023-12-10T18:49:53.491Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.783527786Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3072023-12-10T18:49:53.495Z{"msg":"[2] e68e8be9-3bec-4457-8eba-a77c2680a871 (8654b076-ab9a-4a46-b323-754eaa738ae3) New WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.783552383Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3082023-12-10T18:49:53.495Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.783593189Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3092023-12-10T18:49:53.495Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:49:51.783614915Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3102023-12-10T18:49:53.495Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.783654391Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3112023-12-10T18:49:53.495Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.783676695Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3122023-12-10T18:49:53.495Z{"msg":"e68e8be9-3bec-4457-8eba-a77c2680a871 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.783717529Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3132023-12-10T18:49:53.495Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.783739499Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3142023-12-10T18:49:53.495Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:51.783758938Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3152023-12-10T18:49:54.516Z{"msg":"[0] e68e8be9-3bec-4457-8eba-a77c2680a871 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.781396196Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"looper":"0","session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3162023-12-10T18:49:54.517Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.781482718Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3172023-12-10T18:49:54.517Z{"msg":"[0] e68e8be9-3bec-4457-8eba-a77c2680a871 (8654b076-ab9a-4a46-b323-754eaa738ae3) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.782605327Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3182023-12-10T18:49:54.517Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.782670794Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3192023-12-10T18:49:54.517Z{"msg":"[0] client is_active_req TRUE, promote! session 8654b076-ab9a-4a46-b323-754eaa738ae3","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.782719087Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3202023-12-10T18:49:54.517Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.783305015Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3212023-12-10T18:49:54.517Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.783362885Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3222023-12-10T18:49:54.517Z{"msg":"e68e8be9-3bec-4457-8eba-a77c2680a871 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.783409272Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3232023-12-10T18:49:54.517Z{"msg":"[0] e68e8be9-3bec-4457-8eba-a77c2680a871 (8654b076-ab9a-4a46-b323-754eaa738ae3) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.784083982Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3242023-12-10T18:49:54.517Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.784139139Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3252023-12-10T18:49:54.517Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:49:52.784163591Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3262023-12-10T18:49:54.518Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.784184285Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3272023-12-10T18:49:54.518Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.784226014Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3282023-12-10T18:49:54.518Z{"msg":"e68e8be9-3bec-4457-8eba-a77c2680a871 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.784269559Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3292023-12-10T18:49:54.518Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.784313239Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3302023-12-10T18:49:54.519Z{"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.784356915Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3312023-12-10T18:49:54.519Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.784384149Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3322023-12-10T18:49:54.519Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:2
3332023-12-10T18:49:54.519ZWait for a query_work_queue command to finish before sending IO
3342023-12-10T18:49:54.519Z,"time":"2023-12-10T18:49:52.784426174Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3352023-12-10T18:49:54.519Z{"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.784572566Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3362023-12-10T18:49:54.519Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.784649592Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3372023-12-10T18:49:54.519Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.784673648Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3382023-12-10T18:49:54.519Z{"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.784704696Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3392023-12-10T18:49:54.519Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.784777929Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3402023-12-10T18:49:54.519Z{"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.784846637Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3412023-12-10T18:49:54.520Z{"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.784918884Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3422023-12-10T18:49:54.520Z{"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.784988377Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3432023-12-10T18:49:54.520Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.78505778Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3442023-12-10T18:49:54.520Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.785139513Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3452023-12-10T18:49:54.520Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.785210697Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3462023-12-10T18:49:54.520Z{"msg":"Set Downstairs and Upstairs active","v":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
3472023-12-10T18:49:54.520Z0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.785369904Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3482023-12-10T18:49:54.520Z{"msg":"e68e8be9-3bec-4457-8eba-a77c2680a871 is now active with session: 8654b076-ab9a-4a46-b323-754eaa738ae3","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.785428124Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3492023-12-10T18:49:54.520Z{"msg":"e68e8be9-3bec-4457-8eba-a77c2680a871 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.785473176Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3502023-12-10T18:49:54.520Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.785515939Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3512023-12-10T18:49:54.521Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.785557378Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3522023-12-10T18:49:54.521Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.785598582Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3532023-12-10T18:49:54.521Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.785643477Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3542023-12-10T18:49:54.522Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.785685729Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3552023-12-10T18:49:54.522Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.785727581Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3562023-12-10T18:49:54.522Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.785769282Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3572023-12-10T18:49:54.522Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.785810927Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3582023-12-10T18:49:54.522Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.785852374Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3592023-12-10T18:49:54.522Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.785874383Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3602023-12-10T18:49:54.522Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:52.785913741Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1167,"session_id":"8654b076-ab9a-4a46-b323-754eaa738ae3"}
3612023-12-10T18:49:54.522ZLoading write count information from file "/tmp/test_repair_verify.data"
3622023-12-10T18:49:54.522ZRun Repair workload
3632023-12-10T18:49:54.522Z01/30 Write block 102 len 20480 data: 2 2 2 2 2
3642023-12-10T18:49:54.522Z02/30 Read block 268 len 8192
3652023-12-10T18:49:54.522Z03/30 Write block 555 len 8192 data: 2 2
3662023-12-10T18:49:54.522Z04/30 Read block 187 len 4096
3672023-12-10T18:49:54.522Z05/30 Write block 171 len 32768 data: 2 2 2 2 2 2 2 2
3682023-12-10T18:49:54.536Z06/30 Read block 440 len 36864
3692023-12-10T18:49:54.537Z07/30 Write block 147 len 4096 data: 2
3702023-12-10T18:49:54.537Z08/30 Read block 436 len 4096
3712023-12-10T18:49:54.537Z09/30 Write block 37 len 36864 data: 2 2 2 2 2 2 2 2 2
3722023-12-10T18:49:54.537Z10/30 Write block 288 len 8192 data: 2 2
3732023-12-10T18:49:54.537Z11/30 Read block 25 len 32768
3742023-12-10T18:49:54.538Z12/30 Flush
3752023-12-10T18:49:54.538Z13/30 Write block 427 len 16384 data: 2 2 2 2
3762023-12-10T18:49:54.538Z14/30 Read block 27 len 32768
3772023-12-10T18:49:54.538Z15/30 Write block 234 len 28672 data: 2 2 2 2 2 2 2
3782023-12-10T18:49:54.538Z16/30 Read block 553 len 4096
3792023-12-10T18:49:54.539Z17/30 Write block 541 len 12288 data: 2 2 2
3802023-12-10T18:49:54.539Z18/30 Read block 125 len 16384
3812023-12-10T18:49:54.539Z19/30 Read block 473 len 40960
3822023-12-10T18:49:54.539Z20/30 Write block 118 len 36864 data: 2 2 2 2 2 2 2 2 2
3832023-12-10T18:49:54.540Z21/30 Read block 189 len 32768
3842023-12-10T18:49:54.540Z22/30 Write block 435 len 28672 data: 2 2 2 2 2 2 2
3852023-12-10T18:49:54.541Z23/30 Read block 227 len 36864
3862023-12-10T18:49:54.542Z24/30 Read block 149 len 24576
3872023-12-10T18:49:54.542Z25/30 Read block 554 len 8192
3882023-12-10T18:49:54.542Z26/30 Read block 112 len 32768
3892023-12-10T18:49:54.543Z27/30 Read block 228 len 20480
3902023-12-10T18:49:54.543Z28/30 Write block 390 len 40960 data: 2 2 2 2 2 2 2 2 2 2
3912023-12-10T18:49:54.543Z29/30 Write block 244 len 32768 data: 2 2 2 2 2 2 2 2
3922023-12-10T18:49:54.543Z30/30 Write block 285 len 32768 data: 2 2 2 3 3 2 2 2
3932023-12-10T18:49:54.545Z----------------------------------------------------------------
3942023-12-10T18:49:54.545Z Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30
3952023-12-10T18:49:54.545ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
3962023-12-10T18:49:54.545Z 1 Acked 1000 Write 5 Done Done Done false
3972023-12-10T18:49:54.546Z 2 Acked 1001 Read 2 Done Done Done false
3982023-12-10T18:49:54.546Z 3 Acked 1002 Write 2 Done Done Done false
3992023-12-10T18:49:54.546Z 4 Acked 1003 Read 1 Done Done Done false
4002023-12-10T18:49:54.546Z 5 Acked 1004 Write 8 Done Done Done false
4012023-12-10T18:49:54.546Z 6 Acked 1005 Read 9 Done Done Done false
4022023-12-10T18:49:54.546Z 7 Acked 1006 Write 1 Sent Done Done false
4032023-12-10T18:49:54.547Z 8 Acked 1007 Read 1 Sent Done Done false
4042023-12-10T18:49:54.547Z 9 Acked 1008 Write 9 Sent Done Done false
4052023-12-10T18:49:54.547Z 10 Acked 1009 Write 2 Sent Done Done false
4062023-12-10T18:49:54.547Z 11 Acked 1010 Read 8 Sent Done Done false
4072023-12-10T18:49:54.547Z 12 Acked 1011 Flush 0 Sent Done Done false
4082023-12-10T18:49:54.547Z 13 Acked 1012 Write 4 Sent Done Done false
4092023-12-10T18:49:54.548Z 14 Acked 1013 Read 8 Sent Done Done false
4102023-12-10T18:49:54.548Z 15 Acked 1014 Write 7 Sent Done Done false
4112023-12-10T18:49:54.548Z 16 Acked 1015 Read 1 Sent Done Done false
4122023-12-10T18:49:54.548Z 17 Acked 1016 Write 3 Sent Done Done false
4132023-12-10T18:49:54.548Z 18 Acked 1017 Read 4 Sent Done Done false
4142023-12-10T18:49:54.548Z 19 Acked 1018 Read 10 Sent Done Done false
4152023-12-10T18:49:54.549Z 20 Acked 1019 Write 9 Sent Done Done false
4162023-12-10T18:49:54.549Z 21 Acked 1020 Read 8 Sent Done Done false
4172023-12-10T18:49:54.549Z 22 Acked 1021 Write 7 Sent Done Done false
4182023-12-10T18:49:54.549Z 23 Acked 1022 Read 9 Sent Done Done false
4192023-12-10T18:49:54.549Z 24 Acked 1023 Read 6 Sent Done Done false
4202023-12-10T18:49:54.549Z 25 Acked 1024 Read 2 Sent Done Done false
4212023-12-10T18:49:54.549Z 26 Acked 1025 Read 8 Sent Done Done false
4222023-12-10T18:49:54.550Z 27 Acked 1026 Read 5 Sent Done Done false
4232023-12-10T18:49:54.550Z 28 Acked 1027 Write 10 Sent Done Done false
4242023-12-10T18:49:54.550Z 29 Acked 1028 Write 8 Sent Sent Done false
4252023-12-10T18:49:54.550Z 30 Acked 1029 Write 8 Sent Sent Sent false
4262023-12-10T18:49:54.550Z STATES DS:0 DS:1 DS:2 TOTAL
4272023-12-10T18:49:54.550Z New 0 0 0 0
4282023-12-10T18:49:54.551Z Sent 24 2 1 27
4292023-12-10T18:49:54.551Z Done 6 28 29 63
4302023-12-10T18:49:54.551Z Skipped 0 0 0 0
4312023-12-10T18:49:54.551Z Error 0 0 0 0
4322023-12-10T18:49:54.551ZLast Flush: 0 1011 1011
4332023-12-10T18:49:54.551ZDownstairs last five completed:
4342023-12-10T18:49:54.551ZUpstairs last five completed: 30 29 28 27 26 25
4352023-12-10T18:49:54.552ZWrote out file "/tmp/test_repair_verify.data"
4362023-12-10T18:49:54.552Z
4372023-12-10T18:49:54.552ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
4382023-12-10T18:49:54.552ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4392023-12-10T18:49:54.552Z 0 000-019 1 1 1 1 1 1 F F F
4402023-12-10T18:49:54.553Z 1 020-039 1 2 2 1 2 2 F F F <---
4412023-12-10T18:49:54.554Z 2 040-059 1 2 2 1 2 2 F F F <---
4422023-12-10T18:49:54.554Z 3 060-079 1 1 1 1 1 1 F F F
4432023-12-10T18:49:54.554Z 4 080-099 1 1 1 1 1 1 F F F
4442023-12-10T18:49:54.555Z 5 100-119 1 2 2 1 2 2 T T T <---
4452023-12-10T18:49:54.555Z 6 120-139 1 1 1 1 1 1 F T T <---
4462023-12-10T18:49:54.555Z 7 140-159 1 2 2 1 2 2 F F F <---
4472023-12-10T18:49:54.555Z 8 160-179 1 2 2 1 2 2 T F F <---
4482023-12-10T18:49:54.556Z 9 180-199 1 1 1 1 1 1 F F F
4492023-12-10T18:49:54.556Z 10 200-219 1 1 1 1 1 1 F F F
4502023-12-10T18:49:54.556Z 11 220-239 1 1 1 1 1 1 F T T <---
4512023-12-10T18:49:54.557Z 12 240-259 1 1 1 1 1 1 F T T <---
4522023-12-10T18:49:54.557Z 13 260-279 1 1 1 1 1 1 F F F
4532023-12-10T18:49:54.557Z 14 280-299 1 2 2 1 2 2 F T T <---
4542023-12-10T18:49:54.558Z 15 300-319 1 1 1 1 1 1 F F F
4552023-12-10T18:49:54.558Z 16 320-339 1 1 1 1 1 1 F F F
4562023-12-10T18:49:54.558Z 17 340-359 1 1 1 1 1 1 F F F
4572023-12-10T18:49:54.558Z 18 360-379 1 1 1 1 1 1 F F F
4582023-12-10T18:49:54.559Z 19 380-399 1 1 1 1 1 1 F T T <---
4592023-12-10T18:49:54.559Z 20 400-419 1 1 1 1 1 1 F F F
4602023-12-10T18:49:54.559Z 21 420-439 1 1 1 1 1 1 F T T <---
4612023-12-10T18:49:54.560Z 22 440-459 1 1 1 1 1 1 F T T <---
4622023-12-10T18:49:54.560Z 23 460-479 1 1 1 1 1 1 F F F
4632023-12-10T18:49:54.560Z 24 480-499 1 1 1 1 1 1 F F F
4642023-12-10T18:49:54.561Z 25 500-519 1 1 1 1 1 1 F F F
4652023-12-10T18:49:54.562Z 26 520-539 1 1 1 1 1 1 F F F
4662023-12-10T18:49:54.562Z 27 540-559 1 2 2 1 2 2 T T T <---
4672023-12-10T18:49:54.563Z 28 560-579 1 1 1 1 1 1 F F F
4682023-12-10T18:49:54.573Z 29 580-599 1 1 1 1 1 1 F F F
4692023-12-10T18:49:54.573ZMax gen: 2, Max flush: 2
4702023-12-10T18:49:54.573ZError: Difference in extent metadata found!
4712023-12-10T18:49:54.573ZA Difference in extent metadata is expected here
4722023-12-10T18:49:54.574ZOn loop 1
4732023-12-10T18:49:54.574Z
4742023-12-10T18:49:54.574ZVerifying data now
4752023-12-10T18:50:00.851ZNo change after verify
4762023-12-10T18:50:00.851ZLoop: 1 Downstairs dump after verify (and repair):
4772023-12-10T18:50:00.874ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4782023-12-10T18:50:00.874Z 0 000-019 1 1 1 1 1 1 F F F
4792023-12-10T18:50:00.874Z 1 020-039 2 2 2 2 2 2 F F F
4802023-12-10T18:50:00.874Z 2 040-059 2 2 2 2 2 2 F F F
4812023-12-10T18:50:00.875Z 3 060-079 1 1 1 1 1 1 F F F
4822023-12-10T18:50:00.875Z 4 080-099 1 1 1 1 1 1 F F F
4832023-12-10T18:50:00.875Z 5 100-119 3 3 3 3 3 3 F F F
4842023-12-10T18:50:00.875Z 6 120-139 3 3 3 3 3 3 F F F
4852023-12-10T18:50:00.875Z 7 140-159 2 2 2 2 2 2 F F F
4862023-12-10T18:50:00.875Z 8 160-179 2 2 2 2 2 2 F F F
4872023-12-10T18:50:00.875Z 9 180-199 1 1 1 1 1 1 F F F
4882023-12-10T18:50:00.875Z 10 200-219 1 1 1 1 1 1 F F F
4892023-12-10T18:50:00.875Z 11 220-239 3 3 3 3 3 3 F F F
4902023-12-10T18:50:00.876Z 12 240-259 3 3 3 3 3 3 F F F
4912023-12-10T18:50:00.876Z 13 260-279 1 1 1 1 1 1 F F F
4922023-12-10T18:50:00.876Z 14 280-299 3 3 3 3 3 3 F F F
4932023-12-10T18:50:00.876Z 15 300-319 1 1 1 1 1 1 F F F
4942023-12-10T18:50:00.876Z 16 320-339 1 1 1 1 1 1 F F F
4952023-12-10T18:50:00.877Z 17 340-359 1 1 1 1 1 1 F F F
4962023-12-10T18:50:00.877Z 18 360-379 1 1 1 1 1 1 F F F
4972023-12-10T18:50:00.878Z 19 380-399 3 3 3 3 3 3 F F F
4982023-12-10T18:50:00.878Z 20 400-419 1 1 1 1 1 1 F F F
4992023-12-10T18:50:00.878Z 21 420-439 3 3 3 3 3 3 F F F
5002023-12-10T18:50:00.878Z 22 440-459 3 3 3 3 3 3 F F F
5012023-12-10T18:50:00.878Z 23 460-479 1 1 1 1 1 1 F F F
5022023-12-10T18:50:00.879Z 24 480-499 1 1 1 1 1 1 F F F
5032023-12-10T18:50:00.879Z 25 500-519 1 1 1 1 1 1 F F F
5042023-12-10T18:50:00.879Z 26 520-539 1 1 1 1 1 1 F F F
5052023-12-10T18:50:00.879Z 27 540-559 3 3 3 3 3 3 F F F
5062023-12-10T18:50:00.879Z 28 560-579 1 1 1 1 1 1 F F F
5072023-12-10T18:50:00.879Z 29 580-599 1 1 1 1 1 1 F F F
5082023-12-10T18:50:00.879ZMax gen: 3, Max flush: 3
5092023-12-10T18:50:00.880Z
5102023-12-10T18:50:00.880ZBegin loop 2
5112023-12-10T18:50:00.880ZDownstairs to restart: 1
5122023-12-10T18:50:00.880Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.258761979Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175}
5132023-12-10T18:50:00.880Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.259182573Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175}
5142023-12-10T18:50:00.881Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.259251649Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175}
5152023-12-10T18:50:00.882Z{"msg":"Crucible 04caaf87-9daf-468e-a08f-f08cfce968d0 has session id: d9b92a65-3467-468f-91c6-bb50f4a7aed3","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.259320447Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5162023-12-10T18:50:00.882Z{Crucible runtime is spawned
5172023-12-10T18:50:00.882Z"msg":"Upstairs opts: Upstairs UUID: 04caaf87-9daf-468e-a08f-f08cfce968d0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.25955547Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5182023-12-10T18:50:00.882Z{"msg":"The guest has requested activation with gen:4
5192023-12-10T18:50:00.882ZCrucible stats registered with UUID: 04caaf87-9daf-468e-a08f-f08cfce968d0","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.25978929Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5202023-12-10T18:50:00.882Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.259838355Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"looper":"2","session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5212023-12-10T18:50:00.883Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.259928098Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3","task":"up_listen"}
5222023-12-10T18:50:00.883Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.260002616Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5232023-12-10T18:50:00.883Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.260095615Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5242023-12-10T18:50:00.883Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.26016766Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"looper":"0","session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5252023-12-10T18:50:00.883Z{"msg":"[2] 04caaf87-9daf-468e-a08f-f08cfce968d0 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.260213267Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"looper":"2","session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5262023-12-10T18:50:00.884Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.260241494Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"looper":"1","session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5272023-12-10T18:50:00.884Z{"msg":"[0] 04caaf87-9daf-468e-a08f-f08cfce968d0 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.260271735Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"looper":"0","session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5282023-12-10T18:50:00.884Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.260312396Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5292023-12-10T18:50:00.884Z{"msg":"04caaf87-9daf-468e-a08f-f08cfce968d0 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.26034809Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5302023-12-10T18:50:00.885Z{"msg":"Set desired generation to :4","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.260383395Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5312023-12-10T18:50:00.885Z{"msg":"[1] 04caaf87-9daf-468e-a08f-f08cfce968d0 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.260430568Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"looper":"1","session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5322023-12-10T18:50:00.885Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.260932171Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5332023-12-10T18:50:00.885Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.261009733Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5342023-12-10T18:50:00.885Z{"msg":"[2] 04caaf87-9daf-468e-a08f-f08cfce968d0 (d9b92a65-3467-468f-91c6-bb50f4a7aed3) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.261190828Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5352023-12-10T18:50:00.885Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.261289419Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5362023-12-10T18:50:00.886Z{"msg":"[2] client is_active_req TRUE, promote! session d9b92a65-3467-468f-91c6-bb50f4a7aed3","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.261335916Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5372023-12-10T18:50:00.887Z{"msg":"[0] 04caaf87-9daf-468e-a08f-f08cfce968d0 (d9b92a65-3467-468f-91c6-bb50f4a7aed3) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.261369795Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5382023-12-10T18:50:00.887Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.261391479Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5392023-12-10T18:50:00.887Z{"msg":"[0] client is_active_req TRUE, promote! session d9b92a65-3467-468f-91c6-bb50f4a7aed3","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.261415098Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5402023-12-10T18:50:00.887Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.261861442Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5412023-12-10T18:50:00.887Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.26194482Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5422023-12-10T18:50:00.887Z{"msg":"04caaf87-9daf-468e-a08f-f08cfce968d0 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.261990186Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5432023-12-10T18:50:00.887Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.262081151Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5442023-12-10T18:50:00.888Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.262186323Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5452023-12-10T18:50:00.888Z{"msg":"04caaf87-9daf-468e-a08f-f08cfce968d0 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.262217233Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5462023-12-10T18:50:00.888Z{"msg":"[2] 04caaf87-9daf-468e-a08f-f08cfce968d0 (d9b92a65-3467-468f-91c6-bb50f4a7aed3) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.262708714Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5472023-12-10T18:50:00.888Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.262788614Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5482023-12-10T18:50:00.888Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:49:59.262832238Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5492023-12-10T18:50:00.889Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.262966184Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5502023-12-10T18:50:00.889Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.263060981Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5512023-12-10T18:50:00.889Z{"msg":"04caaf87-9daf-468e-a08f-f08cfce968d0 WaitActive New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.263087516Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5522023-12-10T18:50:00.889Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.263108088Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5532023-12-10T18:50:00.889Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.263130869Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5542023-12-10T18:50:00.889Z{"msg":"[1] 04caaf87-9daf-468e-a08f-f08cfce968d0 (d9b92a65-3467-468f-91c6-bb50f4a7aed3) WaitActive New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.263154417Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5552023-12-10T18:50:00.891Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.263174759Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5562023-12-10T18:50:00.891Z{"msg":"[1] client is_active_req TRUE, promote! session d9b92a65-3467-468f-91c6-bb50f4a7aed3","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.26321735Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5572023-12-10T18:50:00.891Z{"msg":"[0] 04caaf87-9daf-468e-a08f-f08cfce968d0 (d9b92a65-3467-468f-91c6-bb50f4a7aed3) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.263242762Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5582023-12-10T18:50:00.891Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.263263496Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5592023-12-10T18:50:00.891Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:49:59.263304273Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5602023-12-10T18:50:00.891Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.263326406Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5612023-12-10T18:50:00.891Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.263346913Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5622023-12-10T18:50:00.892Z{"msg":"04caaf87-9daf-468e-a08f-f08cfce968d0 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.263388529Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5632023-12-10T18:50:00.892Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.26341107Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5642023-12-10T18:50:00.892Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.26343067Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5652023-12-10T18:50:00.892Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.263843783Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5662023-12-10T18:50:00.892Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.263903189Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5672023-12-10T18:50:00.893Z{"msg":"04caaf87-9daf-468e-a08f-f08cfce968d0 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.263928921Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5682023-12-10T18:50:00.893Z{"msg":"[1] 04caaf87-9daf-468e-a08f-f08cfce968d0 (d9b92a65-3467-468f-91c6-bb50f4a7aed3) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.264498262Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5692023-12-10T18:50:00.893Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.264633066Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5702023-12-10T18:50:00.893Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:49:59.264697733Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5712023-12-10T18:50:00.894Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.264722888Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5722023-12-10T18:50:00.894Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.264772509Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5732023-12-10T18:50:00.894Z{"msg":"04caaf87-9daf-468e-a08f-f08cfce968d0 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.264795856Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal"The guest has finished waiting for activation with:4
5742023-12-10T18:50:00.895ZWait for a query_work_queue command to finish before sending IO
5752023-12-10T18:50:00.895Z,"pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5762023-12-10T18:50:00.895Z{"msg":"[0]R flush_numbers[0..12]: [1, 2, 2, 1, 1, 3, 3, 2, 2, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.264958117Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5772023-12-10T18:50:00.936Z{"msg":"[0]R generation[0..12]: [1, 2, 2, 1, 1, 3, 3, 2, 2, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.265070415Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5782023-12-10T18:50:00.936Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.265098293Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5792023-12-10T18:50:00.936Z{"msg":"[1]R flush_numbers[0..12]: [1, 2, 2, 1, 1, 3, 3, 2, 2, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.265119823Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5802023-12-10T18:50:00.936Z{"msg":"[1]R generation[0..12]: [1, 2, 2, 1, 1, 3, 3, 2, 2, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.265202988Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5812023-12-10T18:50:00.936Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.265231307Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5822023-12-10T18:50:00.936Z{"msg":"[2]R flush_numbers[0..12]: [1, 2, 2, 1, 1, 3, 3, 2, 2, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.265252033Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5832023-12-10T18:50:00.937Z{"msg":"[2]R generation[0..12]: [1, 2, 2, 1, 1, 3, 3, 2, 2, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.265277523Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5842023-12-10T18:50:00.937Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.265338674Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5852023-12-10T18:50:00.937Z{"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.265363745Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5862023-12-10T18:50:00.937Z{"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.265384108Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5872023-12-10T18:50:00.937Z{"msg":"Next flush: 4","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.265415193Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5882023-12-10T18:50:00.938Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.26546792Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5892023-12-10T18:50:00.938Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.265491692Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5902023-12-10T18:50:00.938Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.265548394Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5912023-12-10T18:50:00.938Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.265577523Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5922023-12-10T18:50:00.938Z{"msg":"04caaf87-9daf-468e-a08f-f08cfce968d0 is now active with session: d9b92a65-3467-468f-91c6-bb50f4a7aed3","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.265637629Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5932023-12-10T18:50:00.938Z{"msg":"04caaf87-9daf-468e-a08f-f08cfce968d0 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.26566769Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5942023-12-10T18:50:00.938Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.265727886Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5952023-12-10T18:50:00.938Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.265752493Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5962023-12-10T18:50:00.938Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.265810119Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5972023-12-10T18:50:00.940Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.265835717Z","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
5982023-12-10T18:50:00.940Zhostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
5992023-12-10T18:50:00.940Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.265966307Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
6002023-12-10T18:50:00.940Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.265994334Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
6012023-12-10T18:50:00.940Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.266051915Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
6022023-12-10T18:50:00.940Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.266081456Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
6032023-12-10T18:50:00.940Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.266136947Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
6042023-12-10T18:50:00.940Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.266166073Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
6052023-12-10T18:50:00.941Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:49:59.266220407Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1175,"session_id":"d9b92a65-3467-468f-91c6-bb50f4a7aed3"}
6062023-12-10T18:50:00.941ZLoading write count information from file "/tmp/test_repair_verify.data"
6072023-12-10T18:50:00.941ZRun Repair workload
6082023-12-10T18:50:00.941Z01/30 Flush
6092023-12-10T18:50:00.941Z02/30 Write block 238 len 32768 data: 3 3 3 2 2 2 3 3
6102023-12-10T18:50:00.941Z03/30 Flush
6112023-12-10T18:50:00.941Z04/30 Write block 506 len 4096 data: 2
6122023-12-10T18:50:00.942Z05/30 Read block 482 len 12288
6132023-12-10T18:50:00.942Z06/30 Read block 216 len 32768
6142023-12-10T18:50:00.942Z07/30 Read block 210 len 12288
6152023-12-10T18:50:00.942Z08/30 Read block 65 len 28672
6162023-12-10T18:50:00.942Z09/30 Read block 419 len 32768
6172023-12-10T18:50:00.942Z10/30 Write block 288 len 4096 data: 4
6182023-12-10T18:50:00.942Z11/30 Write block 74 len 8192 data: 2 2
6192023-12-10T18:50:00.942Z12/30 Flush
6202023-12-10T18:50:00.942Z13/30 Write block 542 len 20480 data: 3 3 2 2 2
6212023-12-10T18:50:00.942Z14/30 Write block 434 len 4096 data: 2
6222023-12-10T18:50:00.942Z15/30 Write block 499 len 40960 data: 2 2 2 2 2 2 2 3 2 2
6232023-12-10T18:50:00.942Z16/30 Write block 273 len 40960 data: 2 2 2 2 2 2 2 2 2 2
6242023-12-10T18:50:00.942Z17/30 Write block 327 len 24576 data: 2 2 2 2 2 2
6252023-12-10T18:50:00.943Z18/30 Read block 378 len 24576
6262023-12-10T18:50:00.943Z19/30 Write block 80 len 16384 data: 2 2 2 2
6272023-12-10T18:50:00.943Z20/30 Read block 527 len 28672
6282023-12-10T18:50:00.943Z21/30 Read block 295 len 4096
6292023-12-10T18:50:00.943Z22/30 Write block 274 len 24576 data: 3 3 3 3 3 3
6302023-12-10T18:50:00.943Z23/30 Write block 438 len 4096 data: 3
6312023-12-10T18:50:00.943Z24/30 Write block 66 len 16384 data: 2 2 2 2
6322023-12-10T18:50:00.943Z25/30 Write block 589 len 36864 data: 2 2 2 2 2 2 2 2 2
6332023-12-10T18:50:00.943Z26/30 Write block 46 len 28672 data: 2 2 2 2 2 2 2
6342023-12-10T18:50:00.943Z27/30 Write block 216 len 4096 data: 2
6352023-12-10T18:50:00.943Z28/30 Read block 96 len 16384
6362023-12-10T18:50:00.943Z29/30 Read block 477 len 16384
6372023-12-10T18:50:00.943Z30/30 Write block 86 len 40960 data: 2 2 2 2 2 2 2 2 2 2
6382023-12-10T18:50:00.943Z----------------------------------------------------------------
6392023-12-10T18:50:00.943Z Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:30
6402023-12-10T18:50:00.943ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
6412023-12-10T18:50:00.944Z 1 Acked 1000 Flush 0 Done Sent Done false
6422023-12-10T18:50:00.944Z 2 Acked 1001 Write 8 Done Sent Done false
6432023-12-10T18:50:00.944Z 3 Acked 1002 Flush 0 Done Sent Done false
6442023-12-10T18:50:00.944Z 4 Acked 1003 Write 1 Done Sent Done false
6452023-12-10T18:50:00.944Z 5 Acked 1004 Read 3 Done Sent Done false
6462023-12-10T18:50:00.944Z 6 Acked 1005 Read 8 Done Sent Done false
6472023-12-10T18:50:00.944Z 7 Acked 1006 Read 3 Done Sent Done false
6482023-12-10T18:50:00.944Z 8 Acked 1007 Read 7 Done Sent Done false
6492023-12-10T18:50:00.944Z 9 Acked 1008 Read 8 Done Sent Done false
6502023-12-10T18:50:00.944Z 10 Acked 1009 Write 1 Done Sent Done false
6512023-12-10T18:50:00.944Z 11 Acked 1010 Write 2 Done Sent Done false
6522023-12-10T18:50:00.944Z 12 Acked 1011 Flush 0 Done Sent Done false
6532023-12-10T18:50:00.944Z 13 Acked 1012 Write 5 Done Sent Done false
6542023-12-10T18:50:00.944Z 14 Acked 1013 Write 1 Done Sent Done false
6552023-12-10T18:50:00.944Z 15 Acked 1014 Write 10 Done Sent Done false
6562023-12-10T18:50:00.944Z 16 Acked 1015 Write 10 Done Sent Done false
6572023-12-10T18:50:00.944Z 17 Acked 1016 Write 6 Done Sent Done false
6582023-12-10T18:50:00.944Z 18 Acked 1017 Read 6 Done Sent Done false
6592023-12-10T18:50:00.945Z 19 Acked 1018 Write 4 Done Sent Done false
6602023-12-10T18:50:00.945Z 20 Acked 1019 Read 7 Done Sent Done false
6612023-12-10T18:50:00.945Z 21 Acked 1020 Read 1 Done Sent Done false
6622023-12-10T18:50:00.945Z 22 Acked 1021 Write 6 Done Sent Done false
6632023-12-10T18:50:00.945Z 23 Acked 1022 Write 1 Done Sent Done false
6642023-12-10T18:50:00.945Z 24 Acked 1023 Write 4 Done Sent Done false
6652023-12-10T18:50:00.945Z 25 Acked 1024 Write 9 Done Sent Done false
6662023-12-10T18:50:00.945Z 26 Acked 1025 Write 7 Done Sent Done false
6672023-12-10T18:50:00.945Z 27 Acked 1026 Write 1 Done Sent Done false
6682023-12-10T18:50:00.945Z 28 Acked 1027 Read 4 Done Sent Done false
6692023-12-10T18:50:00.945Z 29 Acked 1028 Read 4 Done Sent Done false
6702023-12-10T18:50:00.945Z 30 Acked 1029 Write 10 Sent Sent Sent false
6712023-12-10T18:50:00.945Z STATES DS:0 DS:1 DS:2 TOTAL
6722023-12-10T18:50:00.945Z New 0 0 0 0
6732023-12-10T18:50:00.945Z Sent 1 30 1 32
6742023-12-10T18:50:00.945Z Done 29 0 29 58
6752023-12-10T18:50:00.945Z Skipped 0 0 0 0
6762023-12-10T18:50:00.946Z Error 0 0 0 0
6772023-12-10T18:50:00.952ZLast Flush: 1011 0 1011
6782023-12-10T18:50:00.952ZDownstairs last five completed:
6792023-12-10T18:50:00.952ZUpstairs last five completed: 30 29 28 27 26 25
6802023-12-10T18:50:00.952ZWrote out file "/tmp/test_repair_verify.data"
6812023-12-10T18:50:00.952Z
6822023-12-10T18:50:00.952ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
6832023-12-10T18:50:00.952ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6842023-12-10T18:50:00.952Z 0 000-019 1 1 1 1 1 1 F F F
6852023-12-10T18:50:00.952Z 1 020-039 2 2 2 2 2 2 F F F
6862023-12-10T18:50:00.952Z 2 040-059 2 2 2 2 2 2 T F T <---
6872023-12-10T18:50:00.952Z 3 060-079 4 1 4 6 1 6 T F T <---
6882023-12-10T18:50:00.952Z 4 080-099 1 1 1 1 1 1 T F T <---
6892023-12-10T18:50:00.952Z 5 100-119 3 3 3 3 3 3 F F F
6902023-12-10T18:50:00.953Z 6 120-139 3 3 3 3 3 3 F F F
6912023-12-10T18:50:00.953Z 7 140-159 2 2 2 2 2 2 F F F
6922023-12-10T18:50:00.953Z 8 160-179 2 2 2 2 2 2 F F F
6932023-12-10T18:50:00.953Z 9 180-199 1 1 1 1 1 1 F F F
6942023-12-10T18:50:00.953Z 10 200-219 1 1 1 1 1 1 T F T <---
6952023-12-10T18:50:00.953Z 11 220-239 4 3 4 5 3 5 F F F <---
6962023-12-10T18:50:00.953Z 12 240-259 4 3 4 5 3 5 F F F <---
6972023-12-10T18:50:00.953Z 13 260-279 1 1 1 1 1 1 T F T <---
6982023-12-10T18:50:00.953Z 14 280-299 4 3 4 6 3 6 T F T <---
6992023-12-10T18:50:00.953Z 15 300-319 1 1 1 1 1 1 F F F
7002023-12-10T18:50:00.953Z 16 320-339 1 1 1 1 1 1 T F T <---
7012023-12-10T18:50:00.953Z 17 340-359 1 1 1 1 1 1 F F F
7022023-12-10T18:50:00.953Z 18 360-379 1 1 1 1 1 1 F F F
7032023-12-10T18:50:00.953Z 19 380-399 3 3 3 3 3 3 F F F
7042023-12-10T18:50:00.953Z 20 400-419 1 1 1 1 1 1 F F F
7052023-12-10T18:50:00.953Z 21 420-439 3 3 3 3 3 3 T F T <---
7062023-12-10T18:50:00.954Z 22 440-459 3 3 3 3 3 3 F F F
7072023-12-10T18:50:00.954Z 23 460-479 1 1 1 1 1 1 F F F
7082023-12-10T18:50:00.954Z 24 480-499 1 1 1 1 1 1 T F T <---
7092023-12-10T18:50:00.954Z 25 500-519 4 1 4 6 1 6 T F T <---
7102023-12-10T18:50:00.954Z 26 520-539 1 1 1 1 1 1 F F F
7112023-12-10T18:50:00.954Z 27 540-559 3 3 3 3 3 3 T F T <---
7122023-12-10T18:50:00.955Z 28 560-579 1 1 1 1 1 1 F F F
7132023-12-10T18:50:00.955Z 29 580-599 1 1 1 1 1 1 T F T <---
7142023-12-10T18:50:00.955ZMax gen: 4, Max flush: 6
7152023-12-10T18:50:00.955ZError: Difference in extent metadata found!
7162023-12-10T18:50:00.955ZA Difference in extent metadata is expected here
7172023-12-10T18:50:00.955ZOn loop 2
7182023-12-10T18:50:00.955Z
7192023-12-10T18:50:00.955ZVerifying data now
7202023-12-10T18:50:05.979ZNo change after verify
7212023-12-10T18:50:05.980ZLoop: 2 Downstairs dump after verify (and repair):
7222023-12-10T18:50:06.006ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
7232023-12-10T18:50:06.006Z 0 000-019 1 1 1 1 1 1 F F F
7242023-12-10T18:50:06.020Z 1 020-039 2 2 2 2 2 2 F F F
7252023-12-10T18:50:06.020Z 2 040-059 5 5 5 7 7 7 F F F
7262023-12-10T18:50:06.021Z 3 060-079 5 5 5 7 7 7 F F F
7272023-12-10T18:50:06.021Z 4 080-099 5 5 5 7 7 7 F F F
7282023-12-10T18:50:06.021Z 5 100-119 3 3 3 3 3 3 F F F
7292023-12-10T18:50:06.022Z 6 120-139 3 3 3 3 3 3 F F F
7302023-12-10T18:50:06.023Z 7 140-159 2 2 2 2 2 2 F F F
7312023-12-10T18:50:06.023Z 8 160-179 2 2 2 2 2 2 F F F
7322023-12-10T18:50:06.024Z 9 180-199 1 1 1 1 1 1 F F F
7332023-12-10T18:50:06.024Z 10 200-219 5 5 5 7 7 7 F F F
7342023-12-10T18:50:06.024Z 11 220-239 4 4 4 5 5 5 F F F
7352023-12-10T18:50:06.024Z 12 240-259 4 4 4 5 5 5 F F F
7362023-12-10T18:50:06.025Z 13 260-279 5 5 5 7 7 7 F F F
7372023-12-10T18:50:06.025Z 14 280-299 5 5 5 7 7 7 F F F
7382023-12-10T18:50:06.025Z 15 300-319 1 1 1 1 1 1 F F F
7392023-12-10T18:50:06.025Z 16 320-339 5 5 5 7 7 7 F F F
7402023-12-10T18:50:06.026Z 17 340-359 1 1 1 1 1 1 F F F
7412023-12-10T18:50:06.026Z 18 360-379 1 1 1 1 1 1 F F F
7422023-12-10T18:50:06.026Z 19 380-399 3 3 3 3 3 3 F F F
7432023-12-10T18:50:06.027Z 20 400-419 1 1 1 1 1 1 F F F
7442023-12-10T18:50:06.028Z 21 420-439 5 5 5 7 7 7 F F F
7452023-12-10T18:50:06.028Z 22 440-459 3 3 3 3 3 3 F F F
7462023-12-10T18:50:06.029Z 23 460-479 1 1 1 1 1 1 F F F
7472023-12-10T18:50:06.029Z 24 480-499 5 5 5 7 7 7 F F F
7482023-12-10T18:50:06.029Z 25 500-519 5 5 5 7 7 7 F F F
7492023-12-10T18:50:06.029Z 26 520-539 1 1 1 1 1 1 F F F
7502023-12-10T18:50:06.029Z 27 540-559 5 5 5 7 7 7 F F F
7512023-12-10T18:50:06.029Z 28 560-579 1 1 1 1 1 1 F F F
7522023-12-10T18:50:06.030Z 29 580-599 5 5 5 7 7 7 F F F
7532023-12-10T18:50:06.030ZMax gen: 5, Max flush: 7
7542023-12-10T18:50:06.030Z
7552023-12-10T18:50:06.031ZBegin loop 3
7562023-12-10T18:50:06.031ZDownstairs to restart: 1
7572023-12-10T18:50:06.046Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.81226522Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183}
7582023-12-10T18:50:06.046Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.812658564Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183}
7592023-12-10T18:50:06.047Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.812689365Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183}
7602023-12-10T18:50:06.047Z{"msg":"Crucible 6416a721-d687-4803-a37a-636c441ad2c3 has session id: e1006344-e9ac-45d8-8145-2637b9aacdd7","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.812733228Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7612023-12-10T18:50:06.069Z{"msg":"Upstairs opts: Upstairs UUID: 6416a721-d687-4803-a37a-636c441ad2c3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.812770693Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7622023-12-10T18:50:06.069Z{"msg":"Crucible stats registered with UUID: 6416a721-d687-4803-a37a-636c441ad2c3","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.81284089Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7632023-12-10T18:50:06.069ZCrucible runtime is spawned
7642023-12-10T18:50:06.070ZThe guest has requested activation with gen:6
7652023-12-10T18:50:06.070Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.81315937Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7","task":"up_listen"}
7662023-12-10T18:50:06.070Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.813263715Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"looper":"0","session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7672023-12-10T18:50:06.073Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.813324474Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"looper":"2","session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7682023-12-10T18:50:06.074Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.813373437Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7692023-12-10T18:50:06.074Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.813450966Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"looper":"1","session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7702023-12-10T18:50:06.074Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.813504617Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7712023-12-10T18:50:06.075Z{"msg":"6416a721-d687-4803-a37a-636c441ad2c3 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.813577417Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7722023-12-10T18:50:06.075Z{"msg":"Set desired generation to :6","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.81361954Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7732023-12-10T18:50:06.075Z{"msg":"[2] 6416a721-d687-4803-a37a-636c441ad2c3 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.813657558Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"looper":"2","session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7742023-12-10T18:50:06.075Z{"msg":"[0] 6416a721-d687-4803-a37a-636c441ad2c3 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.813745702Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"looper":"0","session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7752023-12-10T18:50:06.075Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.813786127Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7762023-12-10T18:50:06.075Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.814206849Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7772023-12-10T18:50:06.076Z{"msg":"[2] 6416a721-d687-4803-a37a-636c441ad2c3 (e1006344-e9ac-45d8-8145-2637b9aacdd7) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.814614633Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7782023-12-10T18:50:06.076Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.814720731Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7792023-12-10T18:50:06.077Z{"msg":"[0] 6416a721-d687-4803-a37a-636c441ad2c3 (e1006344-e9ac-45d8-8145-2637b9aacdd7) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.814755429Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7802023-12-10T18:50:06.077Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.814839196Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7812023-12-10T18:50:06.077Z{"msg":"[2] client is_active_req TRUE, promote! session e1006344-e9ac-45d8-8145-2637b9aacdd7","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.81487484Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7822023-12-10T18:50:06.078Z{"msg":"[0] client is_active_req TRUE, promote! session e1006344-e9ac-45d8-8145-2637b9aacdd7","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.814909952Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7832023-12-10T18:50:06.078Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.815454081Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7842023-12-10T18:50:06.078Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.815539823Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7852023-12-10T18:50:06.078Z{"msg":"6416a721-d687-4803-a37a-636c441ad2c3 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.815583605Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7862023-12-10T18:50:06.079Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.815670026Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7872023-12-10T18:50:06.081Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.815718616Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7882023-12-10T18:50:06.081Z{"msg":"6416a721-d687-4803-a37a-636c441ad2c3 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.815787563Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7892023-12-10T18:50:06.081Z{"msg":"[2] 6416a721-d687-4803-a37a-636c441ad2c3 (e1006344-e9ac-45d8-8145-2637b9aacdd7) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.81640078Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7902023-12-10T18:50:06.081Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.816442656Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7912023-12-10T18:50:06.082Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:04.816466068Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7922023-12-10T18:50:06.082Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.816486393Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7932023-12-10T18:50:06.082Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.816532364Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7942023-12-10T18:50:06.082Z{"msg":"6416a721-d687-4803-a37a-636c441ad2c3 WaitActive New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.816566943Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7952023-12-10T18:50:06.083Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.81660772Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7962023-12-10T18:50:06.083Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.816643986Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7972023-12-10T18:50:06.084Z{"msg":"[0] 6416a721-d687-4803-a37a-636c441ad2c3 (e1006344-e9ac-45d8-8145-2637b9aacdd7) WaitActive New WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.816705368Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7982023-12-10T18:50:06.084Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.816744457Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
7992023-12-10T18:50:06.084Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:04.816805946Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8002023-12-10T18:50:06.084Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.816831028Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8012023-12-10T18:50:06.084Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.816862304Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8022023-12-10T18:50:06.084Z{"msg":"6416a721-d687-4803-a37a-636c441ad2c3 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.816918465Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8032023-12-10T18:50:06.084Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.816947188Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8042023-12-10T18:50:06.084Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:04.817002671Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8052023-12-10T18:50:07.106Z{"msg":"[1] 6416a721-d687-4803-a37a-636c441ad2c3 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.814746546Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"looper":"1","session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8062023-12-10T18:50:07.106Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.814836307Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8072023-12-10T18:50:07.115Z{"msg":"[1] 6416a721-d687-4803-a37a-636c441ad2c3 (e1006344-e9ac-45d8-8145-2637b9aacdd7) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.8158475Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8082023-12-10T18:50:07.116Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.815908041Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8092023-12-10T18:50:07.116Z{"msg":"[1] client is_active_req TRUE, promote! session e1006344-e9ac-45d8-8145-2637b9aacdd7","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.815934646Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8102023-12-10T18:50:07.116Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.816515648Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8112023-12-10T18:50:07.116Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.816573459Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8122023-12-10T18:50:07.116Z{"msg":"6416a721-d687-4803-a37a-636c441ad2c3 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.816599402Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8132023-12-10T18:50:07.116Z{"msg":"[1] 6416a721-d687-4803-a37a-636c441ad2c3 (e1006344-e9ac-45d8-8145-2637b9aacdd7) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.817255427Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8142023-12-10T18:50:07.116Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.817307512Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8152023-12-10T18:50:07.116Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:05.817332053Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8162023-12-10T18:50:07.116Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.817352613Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8172023-12-10T18:50:07.117Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.817373582Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8182023-12-10T18:50:07.117Z{"msg":"6416a721-d687-4803-a37a-636c441ad2c3 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.817395046Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8192023-12-10T18:50:07.117Z{"msg":"[0]R flush_numbers[0..12]: [1, 2, 7, 7, 7, 3, 3, 2, 2, 1, 7, 5]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.817416494Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8202023-12-10T18:50:07.117Z{"msg":"[0]R generation[0..12]: [1, 2, 5, 5, 5, 3, 3, 2, 2, 1, 5, 4]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.81743747Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8212023-12-10T18:50:07.117Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.817460092Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8222023-12-10T18:50:07.117Z{"msg":"[1]R flush_numbers[0..12]: [1, 2, 7, 7, 7, 3, 3, 2, 2, 1, 7, 5]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.817501967Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8232023-12-10T18:50:07.117Z{"msg":"[1]R generation[0..12]: [1, 2, 5, 5, 5, 3, 3, 2, 2, 1, 5, 4]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.817525744Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8242023-12-10T18:50:07.117Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.817578357Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8252023-12-10T18:50:07.117Z{"msg":"[2]R flush_numbers[0..12]: [1, 2, 7, 7, 7, 3, 3, 2, 2, 1, 7, 5]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.817601694Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"The guest has finished waiting for activation with:6
8262023-12-10T18:50:07.117ZWait for a query_work_queue command to finish before sending IO
8272023-12-10T18:50:07.117Ze1006344-e9ac-45d8-8145-2637b9aacdd7"}
8282023-12-10T18:50:07.118Z{"msg":"[2]R generation[0..12]: [1, 2, 5, 5, 5, 3, 3, 2, 2, 1, 5, 4]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.817707294Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8292023-12-10T18:50:07.118Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.817734235Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8302023-12-10T18:50:07.118Z{"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.817781141Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8312023-12-10T18:50:07.118Z{"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.817804528Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8322023-12-10T18:50:07.119Z{"msg":"Next flush: 8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.817825263Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8332023-12-10T18:50:07.119Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.817899377Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8342023-12-10T18:50:07.119Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.81794005Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8352023-12-10T18:50:07.119Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.817978766Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8362023-12-10T18:50:07.119Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.818016749Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8372023-12-10T18:50:07.119Z{"msg":"6416a721-d687-4803-a37a-636c441ad2c3 is now active with session: e1006344-e9ac-45d8-8145-2637b9aacdd7","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.818056239Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8382023-12-10T18:50:07.119Z{"msg":"6416a721-d687-4803-a37a-636c441ad2c3 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.818118054Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8392023-12-10T18:50:07.125Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.818159633Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8402023-12-10T18:50:07.125Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.81821542Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8412023-12-10T18:50:07.125Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.818255262Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8422023-12-10T18:50:07.125Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.818326589Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8432023-12-10T18:50:07.125Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.818366113Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8442023-12-10T18:50:07.125Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.818436981Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8452023-12-10T18:50:07.125Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.818476114Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8462023-12-10T18:50:07.125Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
8472023-12-10T18:50:07.125Z,"time":"2023-12-10T18:50:05.818542826Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8482023-12-10T18:50:07.125Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.818590593Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8492023-12-10T18:50:07.125Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.81864532Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8502023-12-10T18:50:07.125Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:05.818684299Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1183,"session_id":"e1006344-e9ac-45d8-8145-2637b9aacdd7"}
8512023-12-10T18:50:07.125ZLoading write count information from file "/tmp/test_repair_verify.data"
8522023-12-10T18:50:07.125ZRun Repair workload
8532023-12-10T18:50:07.125Z01/30 Write block 181 len 20480 data: 2 2 2 2 2
8542023-12-10T18:50:07.126Z02/30 Read block 353 len 36864
8552023-12-10T18:50:07.126Z03/30 Read block 329 len 16384
8562023-12-10T18:50:07.126Z04/30 Flush
8572023-12-10T18:50:07.126Z05/30 Write block 116 len 36864 data: 2 2 3 3 3 3 3 3 3
8582023-12-10T18:50:07.126Z06/30 Write block 169 len 36864 data: 2 2 3 3 3 3 3 3 3
8592023-12-10T18:50:07.126Z07/30 Read block 266 len 16384
8602023-12-10T18:50:07.126Z08/30 Read block 261 len 8192
8612023-12-10T18:50:07.126Z09/30 Write block 565 len 28672 data: 2 2 2 2 2 2 2
8622023-12-10T18:50:07.126Z10/30 Read block 390 len 24576
8632023-12-10T18:50:07.126Z11/30 Read block 531 len 40960
8642023-12-10T18:50:07.126Z12/30 Read block 479 len 8192
8652023-12-10T18:50:07.126Z13/30 Write block 380 len 12288 data: 2 2 2
8662023-12-10T18:50:07.126Z14/30 Read block 48 len 8192
8672023-12-10T18:50:07.126Z15/30 Read block 353 len 12288
8682023-12-10T18:50:07.126Z16/30 Read block 278 len 12288
8692023-12-10T18:50:07.126Z17/30 Read block 558 len 32768
8702023-12-10T18:50:07.126Z18/30 Write block 284 len 20480 data: 2 3 3 3 5
8712023-12-10T18:50:07.126Z19/30 Write block 236 len 20480 data: 3 3 4 4 4
8722023-12-10T18:50:07.127Z20/30 Read block 205 len 8192
8732023-12-10T18:50:07.127Z21/30 Read block 277 len 40960
8742023-12-10T18:50:07.127Z22/30 Read block 596 len 4096
8752023-12-10T18:50:07.127Z23/30 Read block 288 len 8192
8762023-12-10T18:50:07.127Z24/30 Write block 359 len 8192 data: 2 2
8772023-12-10T18:50:07.127Z25/30 Read block 200 len 16384
8782023-12-10T18:50:07.127Z26/30 Read block 35 len 40960
8792023-12-10T18:50:07.127Z27/30 Write block 389 len 32768 data: 2 3 3 3 3 3 3 3
8802023-12-10T18:50:07.127Z28/30 Write block 160 len 16384 data: 2 2 2 2
8812023-12-10T18:50:07.127Z29/30 Write block 367 len 16384 data: 2 2 2 2
8822023-12-10T18:50:07.127Z30/30 Write block 356 len 8192 data: 2 2
8832023-12-10T18:50:07.127Z----------------------------------------------------------------
8842023-12-10T18:50:07.127Z Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30
8852023-12-10T18:50:07.127ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
8862023-12-10T18:50:07.127Z 1 Acked 1000 Write 5 Done Done Done false
8872023-12-10T18:50:07.127Z 2 Acked 1001 Read 9 Done Done Done false
8882023-12-10T18:50:07.128Z 3 Acked 1002 Read 4 Done Sent Done false
8892023-12-10T18:50:07.128Z 4 Acked 1003 Flush 0 Done Sent Done false
8902023-12-10T18:50:07.128Z 5 Acked 1004 Write 9 Done Sent Done false
8912023-12-10T18:50:07.128Z 6 Acked 1005 Write 9 Done Sent Done false
8922023-12-10T18:50:07.128Z 7 Acked 1006 Read 4 Done Sent Done false
8932023-12-10T18:50:07.128Z 8 Acked 1007 Read 2 Done Sent Done false
8942023-12-10T18:50:07.128Z 9 Acked 1008 Write 7 Done Sent Done false
8952023-12-10T18:50:07.128Z 10 Acked 1009 Read 6 Done Sent Done false
8962023-12-10T18:50:07.128Z 11 Acked 1010 Read 10 Done Sent Done false
8972023-12-10T18:50:07.128Z 12 Acked 1011 Read 2 Done Sent Done false
8982023-12-10T18:50:07.128Z 13 Acked 1012 Write 3 Done Sent Done false
8992023-12-10T18:50:07.128Z 14 Acked 1013 Read 2 Done Sent Done false
9002023-12-10T18:50:07.129Z 15 Acked 1014 Read 3 Done Sent Done false
9012023-12-10T18:50:07.129Z 16 Acked 1015 Read 3 Done Sent Done false
9022023-12-10T18:50:07.129Z 17 Acked 1016 Read 8 Done Sent Done false
9032023-12-10T18:50:07.129Z 18 Acked 1017 Write 5 Done Sent Done false
9042023-12-10T18:50:07.129Z 19 Acked 1018 Write 5 Done Sent Done false
9052023-12-10T18:50:07.129Z 20 Acked 1019 Read 2 Done Sent Done false
9062023-12-10T18:50:07.129Z 21 Acked 1020 Read 10 Done Sent Done false
9072023-12-10T18:50:07.134Z 22 Acked 1021 Read 1 Done Sent Done false
9082023-12-10T18:50:07.134Z 23 Acked 1022 Read 2 Done Sent Done false
9092023-12-10T18:50:07.134Z 24 Acked 1023 Write 2 Done Sent Done false
9102023-12-10T18:50:07.134Z 25 Acked 1024 Read 4 Done Sent Done false
9112023-12-10T18:50:07.134Z 26 Acked 1025 Read 10 Done Sent Done false
9122023-12-10T18:50:07.134Z 27 Acked 1026 Write 8 Done Sent Done false
9132023-12-10T18:50:07.134Z 28 Acked 1027 Write 4 Done Sent Done false
9142023-12-10T18:50:07.134Z 29 Acked 1028 Write 4 Done Sent Done false
9152023-12-10T18:50:07.134Z 30 Acked 1029 Write 2 Sent New Sent false
9162023-12-10T18:50:07.134Z STATES DS:0 DS:1 DS:2 TOTAL
9172023-12-10T18:50:07.135Z New 0 1 0 1
9182023-12-10T18:50:07.135Z Sent 1 27 1 29
9192023-12-10T18:50:07.135Z Done 29 2 29 60
9202023-12-10T18:50:07.135Z Skipped 0 0 0 0
9212023-12-10T18:50:07.135Z Error 0 0 0 0
9222023-12-10T18:50:07.135ZLast Flush: 1003 0 1003
9232023-12-10T18:50:07.135ZDownstairs last five completed:
9242023-12-10T18:50:07.135ZUpstairs last five completed: 30 29 28 27 26 25
9252023-12-10T18:50:07.135ZWrote out file "/tmp/test_repair_verify.data"
9262023-12-10T18:50:07.135Z
9272023-12-10T18:50:07.135ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
9282023-12-10T18:50:07.218ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
9292023-12-10T18:50:07.218Z 0 000-019 1 1 1 1 1 1 F F F
9302023-12-10T18:50:07.218Z 1 020-039 2 2 2 2 2 2 F F F
9312023-12-10T18:50:07.218Z 2 040-059 5 5 5 7 7 7 F F F
9322023-12-10T18:50:07.243Z 3 060-079 5 5 5 7 7 7 F F F
9332023-12-10T18:50:07.243Z 4 080-099 5 5 5 7 7 7 F F F
9342023-12-10T18:50:07.243Z 5 100-119 3 3 3 3 3 3 T F T <---
9352023-12-10T18:50:07.243Z 6 120-139 3 3 3 3 3 3 T F T <---
9362023-12-10T18:50:07.243Z 7 140-159 2 2 2 2 2 2 F F F
9372023-12-10T18:50:07.244Z 8 160-179 2 2 2 2 2 2 T F T <---
9382023-12-10T18:50:07.244Z 9 180-199 6 1 6 8 1 8 F T F <---
9392023-12-10T18:50:07.244Z 10 200-219 5 5 5 7 7 7 F F F
9402023-12-10T18:50:07.244Z 11 220-239 4 4 4 5 5 5 T F T <---
9412023-12-10T18:50:07.244Z 12 240-259 4 4 4 5 5 5 T F T <---
9422023-12-10T18:50:07.244Z 13 260-279 5 5 5 7 7 7 F F F
9432023-12-10T18:50:07.244Z 14 280-299 5 5 5 7 7 7 T F T <---
9442023-12-10T18:50:07.244Z 15 300-319 1 1 1 1 1 1 F F F
9452023-12-10T18:50:07.245Z 16 320-339 5 5 5 7 7 7 F F F
9462023-12-10T18:50:07.245Z 17 340-359 1 1 1 1 1 1 T F T <---
9472023-12-10T18:50:07.245Z 18 360-379 1 1 1 1 1 1 T F T <---
9482023-12-10T18:50:07.245Z 19 380-399 3 3 3 3 3 3 T F T <---
9492023-12-10T18:50:07.245Z 20 400-419 1 1 1 1 1 1 F F F
9502023-12-10T18:50:07.245Z 21 420-439 5 5 5 7 7 7 F F F
9512023-12-10T18:50:07.245Z 22 440-459 3 3 3 3 3 3 F F F
9522023-12-10T18:50:07.245Z 23 460-479 1 1 1 1 1 1 F F F
9532023-12-10T18:50:07.245Z 24 480-499 5 5 5 7 7 7 F F F
9542023-12-10T18:50:07.245Z 25 500-519 5 5 5 7 7 7 F F F
9552023-12-10T18:50:07.246Z 26 520-539 1 1 1 1 1 1 F F F
9562023-12-10T18:50:07.246Z 27 540-559 5 5 5 7 7 7 F F F
9572023-12-10T18:50:07.246Z 28 560-579 1 1 1 1 1 1 T F T <---
9582023-12-10T18:50:07.246Z 29 580-599 5 5 5 7 7 7 F F F
9592023-12-10T18:50:07.246ZMax gen: 6, Max flush: 8
9602023-12-10T18:50:07.246ZError: Difference in extent metadata found!
9612023-12-10T18:50:07.246ZA Difference in extent metadata is expected here
9622023-12-10T18:50:07.246ZOn loop 3
9632023-12-10T18:50:07.249Z
9642023-12-10T18:50:07.249ZVerifying data now
9652023-12-10T18:50:12.304ZNo change after verify
9662023-12-10T18:50:12.304ZLoop: 3 Downstairs dump after verify (and repair):
9672023-12-10T18:50:12.327ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
9682023-12-10T18:50:12.327Z 0 000-019 1 1 1 1 1 1 F F F
9692023-12-10T18:50:12.327Z 1 020-039 2 2 2 2 2 2 F F F
9702023-12-10T18:50:12.327Z 2 040-059 5 5 5 7 7 7 F F F
9712023-12-10T18:50:12.330Z 3 060-079 5 5 5 7 7 7 F F F
9722023-12-10T18:50:12.330Z 4 080-099 5 5 5 7 7 7 F F F
9732023-12-10T18:50:12.331Z 5 100-119 7 7 7 9 9 9 F F F
9742023-12-10T18:50:12.331Z 6 120-139 7 7 7 9 9 9 F F F
9752023-12-10T18:50:12.331Z 7 140-159 2 2 2 2 2 2 F F F
9762023-12-10T18:50:12.331Z 8 160-179 7 7 7 9 9 9 F F F
9772023-12-10T18:50:12.331Z 9 180-199 6 6 6 8 8 8 F F F
9782023-12-10T18:50:12.331Z 10 200-219 5 5 5 7 7 7 F F F
9792023-12-10T18:50:12.331Z 11 220-239 7 7 7 9 9 9 F F F
9802023-12-10T18:50:12.331Z 12 240-259 7 7 7 9 9 9 F F F
9812023-12-10T18:50:12.331Z 13 260-279 5 5 5 7 7 7 F F F
9822023-12-10T18:50:12.331Z 14 280-299 7 7 7 9 9 9 F F F
9832023-12-10T18:50:12.331Z 15 300-319 1 1 1 1 1 1 F F F
9842023-12-10T18:50:12.331Z 16 320-339 5 5 5 7 7 7 F F F
9852023-12-10T18:50:12.331Z 17 340-359 7 7 7 9 9 9 F F F
9862023-12-10T18:50:12.331Z 18 360-379 7 7 7 9 9 9 F F F
9872023-12-10T18:50:12.331Z 19 380-399 7 7 7 9 9 9 F F F
9882023-12-10T18:50:12.331Z 20 400-419 1 1 1 1 1 1 F F F
9892023-12-10T18:50:12.331Z 21 420-439 5 5 5 7 7 7 F F F
9902023-12-10T18:50:12.331Z 22 440-459 3 3 3 3 3 3 F F F
9912023-12-10T18:50:12.332Z 23 460-479 1 1 1 1 1 1 F F F
9922023-12-10T18:50:12.332Z 24 480-499 5 5 5 7 7 7 F F F
9932023-12-10T18:50:12.332Z 25 500-519 5 5 5 7 7 7 F F F
9942023-12-10T18:50:12.332Z 26 520-539 1 1 1 1 1 1 F F F
9952023-12-10T18:50:12.332Z 27 540-559 5 5 5 7 7 7 F F F
9962023-12-10T18:50:12.332Z 28 560-579 7 7 7 9 9 9 F F F
9972023-12-10T18:50:12.332Z 29 580-599 5 5 5 7 7 7 F F F
9982023-12-10T18:50:12.332ZMax gen: 7, Max flush: 9
9992023-12-10T18:50:12.335Z
10002023-12-10T18:50:12.335ZBegin loop 4
10012023-12-10T18:50:12.335ZDownstairs to restart: 0
10022023-12-10T18:50:12.364Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.135637758Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191}
10032023-12-10T18:50:12.365Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.136065959Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191}
10042023-12-10T18:50:12.374Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.136099844Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191}
10052023-12-10T18:50:12.374Z{"msg":"Crucible eb0bd277-172b-4087-a4c3-63303e40c7b7 has session id: 739d771f-79b7-4e85-9dd7-8ae4ca104d71","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.136173746Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10062023-12-10T18:50:12.374Z{"msg":"Upstairs opts: Upstairs UUID: eb0bd277-172b-4087-a4c3-63303e40c7b7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.136228539Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10072023-12-10T18:50:12.374Z{"msg":"Crucible stats registered with UUID: eb0bd277-172b-4087-a4c3-63303e40c7b7","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.136270657Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10082023-12-10T18:50:12.375ZCrucible runtime is spawned
10092023-12-10T18:50:12.375Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30The guest has requested activation with gen:8
10102023-12-10T18:50:12.375Z,"time":"2023-12-10T18:50:11.136537382Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71","task":"up_listen"}
10112023-12-10T18:50:12.375Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.13668386Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"looper":"2","session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10122023-12-10T18:50:12.375Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.13672321Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10132023-12-10T18:50:12.375Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.136781474Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10142023-12-10T18:50:12.376Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.136833844Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"looper":"1","session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10152023-12-10T18:50:12.376Z{"msg":"[2] eb0bd277-172b-4087-a4c3-63303e40c7b7 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.136919631Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"looper":"2","session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10162023-12-10T18:50:12.376Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.13709515Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"looper":"0","session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10172023-12-10T18:50:12.376Z{"msg":"[1] eb0bd277-172b-4087-a4c3-63303e40c7b7 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.137178171Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"looper":"1","session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10182023-12-10T18:50:12.376Z{"msg":"[0] eb0bd277-172b-4087-a4c3-63303e40c7b7 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.137229027Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"looper":"0","session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10192023-12-10T18:50:12.376Z{"msg":"eb0bd277-172b-4087-a4c3-63303e40c7b7 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.137444982Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10202023-12-10T18:50:12.376Z{"msg":"Set desired generation to :8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.137511213Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10212023-12-10T18:50:12.376Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.137555227Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10222023-12-10T18:50:12.376Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.137887194Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10232023-12-10T18:50:12.376Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.137973668Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10242023-12-10T18:50:12.377Z{"msg":"[2] eb0bd277-172b-4087-a4c3-63303e40c7b7 (739d771f-79b7-4e85-9dd7-8ae4ca104d71) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.138383112Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10252023-12-10T18:50:12.377Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.138491319Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10262023-12-10T18:50:12.377Z{"msg":"[2] client is_active_req TRUE, promote! session 739d771f-79b7-4e85-9dd7-8ae4ca104d71","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.138535866Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10272023-12-10T18:50:12.377Z{"msg":"[1] eb0bd277-172b-4087-a4c3-63303e40c7b7 (739d771f-79b7-4e85-9dd7-8ae4ca104d71) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.138573213Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10282023-12-10T18:50:12.377Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.138660873Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10292023-12-10T18:50:12.377Z{"msg":"[1] client is_active_req TRUE, promote! session 739d771f-79b7-4e85-9dd7-8ae4ca104d71","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.138704758Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10302023-12-10T18:50:12.377Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.139209403Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10312023-12-10T18:50:12.377Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.139286147Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10322023-12-10T18:50:12.377Z{"msg":"eb0bd277-172b-4087-a4c3-63303e40c7b7 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.139315374Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10332023-12-10T18:50:12.377Z{"msg":"[0] eb0bd277-172b-4087-a4c3-63303e40c7b7 (739d771f-79b7-4e85-9dd7-8ae4ca104d71) New WaitActive WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.139460573Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10342023-12-10T18:50:12.378Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.139529014Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10352023-12-10T18:50:12.378Z{"msg":"[0] client is_active_req TRUE, promote! session 739d771f-79b7-4e85-9dd7-8ae4ca104d71","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.139672408Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10362023-12-10T18:50:12.378Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.139758377Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10372023-12-10T18:50:12.378Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.139805051Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10382023-12-10T18:50:12.379Z{"msg":"eb0bd277-172b-4087-a4c3-63303e40c7b7 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.13987249Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10392023-12-10T18:50:12.379Z{"msg":"[2] eb0bd277-172b-4087-a4c3-63303e40c7b7 (739d771f-79b7-4e85-9dd7-8ae4ca104d71) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.140079604Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10402023-12-10T18:50:12.379Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.140148462Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10412023-12-10T18:50:12.379Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:11.140189451Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10422023-12-10T18:50:12.379Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.140246931Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10432023-12-10T18:50:12.379Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.140287331Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10442023-12-10T18:50:12.380Z{"msg":"eb0bd277-172b-4087-a4c3-63303e40c7b7 WaitActive WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.140326553Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10452023-12-10T18:50:12.380Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.140390775Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10462023-12-10T18:50:12.380Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.140430124Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10472023-12-10T18:50:12.380Z{"msg":"[1] eb0bd277-172b-4087-a4c3-63303e40c7b7 (739d771f-79b7-4e85-9dd7-8ae4ca104d71) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.140595202Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10482023-12-10T18:50:12.380Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.140646929Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10492023-12-10T18:50:12.380Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:11.140670302Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10502023-12-10T18:50:12.380Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.140691513Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10512023-12-10T18:50:12.380Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.140754121Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10522023-12-10T18:50:12.380Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.14077966Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10532023-12-10T18:50:12.380Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.140836712Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10542023-12-10T18:50:12.380Z{"msg":"eb0bd277-172b-4087-a4c3-63303e40c7b7 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.140862582Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10552023-12-10T18:50:12.380Z{"msg":"eb0bd277-172b-4087-a4c3-63303e40c7b7 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.14089018Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10562023-12-10T18:50:12.381Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.140943578Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10572023-12-10T18:50:12.381Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.140967059Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10582023-12-10T18:50:12.381Z{"msg":"[0] eb0bd277-172b-4087-a4c3-63303e40c7b7 (739d771f-79b7-4e85-9dd7-8ae4ca104d71) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.141361584Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10592023-12-10T18:50:12.381Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.141428077Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10602023-12-10T18:50:12.381Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:11.141454065Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10612023-12-10T18:50:12.381Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.141473883Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10622023-12-10T18:50:12.381Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.141497987Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10632023-12-10T18:50:12.381Z{"msg":"eb0bd277-172b-4087-a4c3-63303e40c7b7 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.141554866Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10642023-12-10T18:50:12.381Z{"msg":"[0]R flush_numbers[0..12]: [1, 2, 7, 7, 7, 9, 9, 2, 9, 8, 7, 9]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.141580706Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10652023-12-10T18:50:12.381Z{"msg":"[0]R generation[0..12]: [1, 2, 5, 5, 5, 7, 7, 2, 7, 6, 5, 7]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.141607566Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10662023-12-10T18:50:12.381Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.141662775Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10672023-12-10T18:50:12.381Z{"msg":"[1]R flush_numbers[0..12]: [1, 2, 7, 7, 7, 9, 9, 2, 9, 8, 7, 9]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.141687495Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10682023-12-10T18:50:12.381Z{"msg":"[1]R generation[0..12]: [1, 2, 5, 5, 5, 7, 7, 2, 7, 6, 5, 7]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.141708047Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10692023-12-10T18:50:12.381Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:8
10702023-12-10T18:50:12.381Z,"time":"Wait for a query_work_queue command to finish before sending IO
10712023-12-10T18:50:12.381Z2023-12-10T18:50:11.141743733Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10722023-12-10T18:50:12.381Z{"msg":"[2]R flush_numbers[0..12]: [1, 2, 7, 7, 7, 9, 9, 2, 9, 8, 7, 9]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.141779066Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10732023-12-10T18:50:12.382Z{"msg":"[2]R generation[0..12]: [1, 2, 5, 5, 5, 7, 7, 2, 7, 6, 5, 7]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.141810839Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10742023-12-10T18:50:12.382Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.141866547Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10752023-12-10T18:50:12.382Z{"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.14188946Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10762023-12-10T18:50:12.382Z{"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.141909569Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10772023-12-10T18:50:12.382Z{"msg":"Next flush: 10","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.141937586Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10782023-12-10T18:50:12.382Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.141991385Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10792023-12-10T18:50:12.382Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.142015007Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10802023-12-10T18:50:12.382Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.142076107Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10812023-12-10T18:50:12.382Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.142100506Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10822023-12-10T18:50:12.382Z{"msg":"eb0bd277-172b-4087-a4c3-63303e40c7b7 is now active with session: 739d771f-79b7-4e85-9dd7-8ae4ca104d71","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.142128101Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10832023-12-10T18:50:12.382Z{"msg":"eb0bd277-172b-4087-a4c3-63303e40c7b7 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.142187454Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10842023-12-10T18:50:12.382Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.142211591Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10852023-12-10T18:50:12.382Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.142238088Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10862023-12-10T18:50:12.382Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.142292611Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10872023-12-10T18:50:12.383Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.142316269Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10882023-12-10T18:50:12.383Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.142335932Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10892023-12-10T18:50:12.383Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.14236045Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10902023-12-10T18:50:12.383Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.1424257Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10912023-12-10T18:50:12.383Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.142450046Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10922023-12-10T18:50:12.383Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.142469598Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10932023-12-10T18:50:12.383Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.142526113Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10942023-12-10T18:50:12.383Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:11.142550273Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1191,"session_id":"739d771f-79b7-4e85-9dd7-8ae4ca104d71"}
10952023-12-10T18:50:12.383ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
10962023-12-10T18:50:12.383ZLoading write count information from file "/tmp/test_repair_verify.data"
10972023-12-10T18:50:12.383ZRun Repair workload
10982023-12-10T18:50:12.387Z01/30 Write block 513 len 32768 data: 2 2 2 2 2 2 2 2
10992023-12-10T18:50:12.388Z02/30 Read block 535 len 8192
11002023-12-10T18:50:12.388Z03/30 Read block 269 len 16384
11012023-12-10T18:50:12.388Z04/30 Read block 178 len 12288
11022023-12-10T18:50:12.389Z05/30 Write block 519 len 36864 data: 3 3 2 2 2 2 2 2 2
11032023-12-10T18:50:12.389Z06/30 Read block 410 len 12288
11042023-12-10T18:50:12.389Z07/30 Flush
11052023-12-10T18:50:12.389Z08/30 Write block 22 len 20480 data: 2 2 2 2 2
11062023-12-10T18:50:12.390Z09/30 Read block 13 len 8192
11072023-12-10T18:50:12.390Z10/30 Write block 261 len 20480 data: 2 2 2 2 2
11082023-12-10T18:50:12.390Z11/30 Write block 526 len 40960 data: 3 3 2 2 2 2 2 2 2 2
11092023-12-10T18:50:12.390Z12/30 Read block 310 len 16384
11102023-12-10T18:50:12.393Z13/30 Write block 388 len 12288 data: 2 3 4
11112023-12-10T18:50:12.393Z14/30 Read block 241 len 16384
11122023-12-10T18:50:12.393Z15/30 Read block 505 len 16384
11132023-12-10T18:50:12.396Z16/30 Read block 471 len 16384
11142023-12-10T18:50:12.396Z17/30 Read block 427 len 16384
11152023-12-10T18:50:12.399Z18/30 Read block 451 len 28672
11162023-12-10T18:50:12.405Z19/30 Write block 227 len 36864 data: 2 2 2 2 2 2 2 3 3
11172023-12-10T18:50:12.405Z20/30 Flush
11182023-12-10T18:50:12.406Z21/30 Write block 509 len 12288 data: 2 2 2
11192023-12-10T18:50:12.409Z22/30 Write block 378 len 40960 data: 2 2 3 3 3 2 2 2 2 2
11202023-12-10T18:50:12.409Z23/30 Write block 347 len 28672 data: 2 2 2 2 2 2 2
11212023-12-10T18:50:12.410Z24/30 Write block 538 len 12288 data: 2 2 2
11222023-12-10T18:50:12.410Z25/30 Read block 165 len 12288
11232023-12-10T18:50:12.413Z26/30 Read block 528 len 36864
11242023-12-10T18:50:12.416Z27/30 Write block 525 len 24576 data: 3 4 4 3 3 3
11252023-12-10T18:50:12.416Z28/30 Write block 424 len 40960 data: 2 2 2 3 3 3 3 2 2 2
11262023-12-10T18:50:12.416Z29/30 Write block 491 len 8192 data: 2 2
11272023-12-10T18:50:12.421Z30/30 Write block 540 len 36864 data: 3 3 4 4 3 3 3 2 2
11282023-12-10T18:50:12.421Z----------------------------------------------------------------
11292023-12-10T18:50:12.421Z Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30
11302023-12-10T18:50:12.421ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
11312023-12-10T18:50:12.421Z 1 Acked 1000 Write 8 Done Done Done false
11322023-12-10T18:50:12.421Z 2 Acked 1001 Read 2 Done Done Done false
11332023-12-10T18:50:12.421Z 3 Acked 1002 Read 4 Done Done Done false
11342023-12-10T18:50:12.421Z 4 Acked 1003 Read 3 Done Done Done false
11352023-12-10T18:50:12.421Z 5 Acked 1004 Write 9 Done Done Done false
11362023-12-10T18:50:12.421Z 6 Acked 1005 Read 3 Done Done Done false
11372023-12-10T18:50:12.421Z 7 Acked 1006 Flush 0 Sent Done Done false
11382023-12-10T18:50:12.421Z 8 Acked 1007 Write 5 Sent Done Done false
11392023-12-10T18:50:12.421Z 9 Acked 1008 Read 2 Sent Done Done false
11402023-12-10T18:50:12.422Z 10 Acked 1009 Write 5 Sent Done Done false
11412023-12-10T18:50:12.422Z 11 Acked 1010 Write 10 Sent Done Done false
11422023-12-10T18:50:12.422Z 12 Acked 1011 Read 4 Sent Done Done false
11432023-12-10T18:50:12.422Z 13 Acked 1012 Write 3 Sent Done Done false
11442023-12-10T18:50:12.422Z 14 Acked 1013 Read 4 Sent Done Done false
11452023-12-10T18:50:12.422Z 15 Acked 1014 Read 4 Sent Done Done false
11462023-12-10T18:50:12.422Z 16 Acked 1015 Read 4 Sent Done Done false
11472023-12-10T18:50:12.422Z 17 Acked 1016 Read 4 Sent Done Done false
11482023-12-10T18:50:12.422Z 18 Acked 1017 Read 7 Sent Done Done false
11492023-12-10T18:50:12.422Z 19 Acked 1018 Write 9 Sent Done Done false
11502023-12-10T18:50:12.422Z 20 Acked 1019 Flush 0 Sent Done Done false
11512023-12-10T18:50:12.422Z 21 Acked 1020 Write 3 Sent Done Done false
11522023-12-10T18:50:12.422Z 22 Acked 1021 Write 10 Sent Done Done false
11532023-12-10T18:50:12.422Z 23 Acked 1022 Write 7 Sent Done Done false
11542023-12-10T18:50:12.422Z 24 Acked 1023 Write 3 Sent Done Done false
11552023-12-10T18:50:12.422Z 25 Acked 1024 Read 3 Sent Done Done false
11562023-12-10T18:50:12.422Z 26 Acked 1025 Read 9 Sent Done Done false
11572023-12-10T18:50:12.423Z 27 Acked 1026 Write 6 Sent Done Done false
11582023-12-10T18:50:12.423Z 28 Acked 1027 Write 10 Sent Done Done false
11592023-12-10T18:50:12.423Z 29 Acked 1028 Write 2 Sent Done Done false
11602023-12-10T18:50:12.423Z 30 Acked 1029 Write 9 Sent Sent Sent false
11612023-12-10T18:50:12.423Z STATES DS:0 DS:1 DS:2 TOTAL
11622023-12-10T18:50:12.423Z New 0 0 0 0
11632023-12-10T18:50:12.423Z Sent 24 1 1 26
11642023-12-10T18:50:12.423Z Done 6 29 29 64
11652023-12-10T18:50:12.423Z Skipped 0 0 0 0
11662023-12-10T18:50:12.423Z Error 0 0 0 0
11672023-12-10T18:50:12.423ZLast Flush: 0 1019 1019
11682023-12-10T18:50:12.423ZDownstairs last five completed:
11692023-12-10T18:50:12.423ZUpstairs last five completed: 30 29 28 27 26 25
11702023-12-10T18:50:12.423ZWrote out file "/tmp/test_repair_verify.data"
11712023-12-10T18:50:12.426Z
11722023-12-10T18:50:12.432ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
11732023-12-10T18:50:12.460ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
11742023-12-10T18:50:12.460Z 0 000-019 1 1 1 1 1 1 F F F
11752023-12-10T18:50:12.460Z 1 020-039 2 8 8 2 11 11 F F F <---
11762023-12-10T18:50:12.460Z 2 040-059 5 5 5 7 7 7 F F F
11772023-12-10T18:50:12.460Z 3 060-079 5 5 5 7 7 7 F F F
11782023-12-10T18:50:12.460Z 4 080-099 5 5 5 7 7 7 F F F
11792023-12-10T18:50:12.460Z 5 100-119 7 7 7 9 9 9 F F F
11802023-12-10T18:50:12.468Z 6 120-139 7 7 7 9 9 9 F F F
11812023-12-10T18:50:12.468Z 7 140-159 2 2 2 2 2 2 F F F
11822023-12-10T18:50:12.468Z 8 160-179 7 7 7 9 9 9 F F F
11832023-12-10T18:50:12.468Z 9 180-199 6 6 6 8 8 8 F F F
11842023-12-10T18:50:12.468Z 10 200-219 5 5 5 7 7 7 F F F
11852023-12-10T18:50:12.469Z 11 220-239 7 8 8 9 11 11 F F F <---
11862023-12-10T18:50:12.469Z 12 240-259 7 7 7 9 9 9 F F F
11872023-12-10T18:50:12.469Z 13 260-279 5 8 8 7 11 11 F F F <---
11882023-12-10T18:50:12.469Z 14 280-299 7 7 7 9 9 9 F F F
11892023-12-10T18:50:12.470Z 15 300-319 1 1 1 1 1 1 F F F
11902023-12-10T18:50:12.470Z 16 320-339 5 5 5 7 7 7 F F F
11912023-12-10T18:50:12.470Z 17 340-359 7 7 7 9 9 9 F T T <---
11922023-12-10T18:50:12.470Z 18 360-379 7 7 7 9 9 9 F T T <---
11932023-12-10T18:50:12.470Z 19 380-399 7 8 8 9 11 11 F T T <---
11942023-12-10T18:50:12.470Z 20 400-419 1 1 1 1 1 1 F F F
11952023-12-10T18:50:12.470Z 21 420-439 5 5 5 7 7 7 F T T <---
11962023-12-10T18:50:12.470Z 22 440-459 3 3 3 3 3 3 F F F
11972023-12-10T18:50:12.470Z 23 460-479 1 1 1 1 1 1 F F F
11982023-12-10T18:50:12.470Z 24 480-499 5 5 5 7 7 7 F T T <---
11992023-12-10T18:50:12.470Z 25 500-519 5 8 8 7 10 10 T T T <---
12002023-12-10T18:50:12.470Z 26 520-539 1 8 8 1 11 11 T T T <---
12012023-12-10T18:50:12.470Z 27 540-559 5 5 5 7 7 7 F T T <---
12022023-12-10T18:50:12.470Z 28 560-579 7 7 7 9 9 9 F F F
12032023-12-10T18:50:12.470Z 29 580-599 5 5 5 7 7 7 F F F
12042023-12-10T18:50:12.470ZMax gen: 8, Max flush: 11
12052023-12-10T18:50:12.470ZError: Difference in extent metadata found!
12062023-12-10T18:50:12.474ZA Difference in extent metadata is expected here
12072023-12-10T18:50:12.474ZOn loop 4
12082023-12-10T18:50:12.474Z
12092023-12-10T18:50:12.474ZVerifying data now
12102023-12-10T18:50:17.592ZNo change after verify
12112023-12-10T18:50:17.592ZLoop: 4 Downstairs dump after verify (and repair):
12122023-12-10T18:50:17.615ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12132023-12-10T18:50:17.615Z 0 000-019 1 1 1 1 1 1 F F F
12142023-12-10T18:50:17.615Z 1 020-039 8 8 8 11 11 11 F F F
12152023-12-10T18:50:17.615Z 2 040-059 5 5 5 7 7 7 F F F
12162023-12-10T18:50:17.615Z 3 060-079 5 5 5 7 7 7 F F F
12172023-12-10T18:50:17.615Z 4 080-099 5 5 5 7 7 7 F F F
12182023-12-10T18:50:17.618Z 5 100-119 7 7 7 9 9 9 F F F
12192023-12-10T18:50:17.618Z 6 120-139 7 7 7 9 9 9 F F F
12202023-12-10T18:50:17.618Z 7 140-159 2 2 2 2 2 2 F F F
12212023-12-10T18:50:17.618Z 8 160-179 7 7 7 9 9 9 F F F
12222023-12-10T18:50:17.618Z 9 180-199 6 6 6 8 8 8 F F F
12232023-12-10T18:50:17.618Z 10 200-219 5 5 5 7 7 7 F F F
12242023-12-10T18:50:17.618Z 11 220-239 8 8 8 11 11 11 F F F
12252023-12-10T18:50:17.619Z 12 240-259 7 7 7 9 9 9 F F F
12262023-12-10T18:50:17.619Z 13 260-279 8 8 8 11 11 11 F F F
12272023-12-10T18:50:17.619Z 14 280-299 7 7 7 9 9 9 F F F
12282023-12-10T18:50:17.619Z 15 300-319 1 1 1 1 1 1 F F F
12292023-12-10T18:50:17.619Z 16 320-339 5 5 5 7 7 7 F F F
12302023-12-10T18:50:17.619Z 17 340-359 9 9 9 12 12 12 F F F
12312023-12-10T18:50:17.619Z 18 360-379 9 9 9 12 12 12 F F F
12322023-12-10T18:50:17.619Z 19 380-399 9 9 9 12 12 12 F F F
12332023-12-10T18:50:17.619Z 20 400-419 1 1 1 1 1 1 F F F
12342023-12-10T18:50:17.619Z 21 420-439 9 9 9 12 12 12 F F F
12352023-12-10T18:50:17.619Z 22 440-459 3 3 3 3 3 3 F F F
12362023-12-10T18:50:17.619Z 23 460-479 1 1 1 1 1 1 F F F
12372023-12-10T18:50:17.620Z 24 480-499 9 9 9 12 12 12 F F F
12382023-12-10T18:50:17.620Z 25 500-519 9 9 9 12 12 12 F F F
12392023-12-10T18:50:17.620Z 26 520-539 9 9 9 12 12 12 F F F
12402023-12-10T18:50:17.620Z 27 540-559 9 9 9 12 12 12 F F F
12412023-12-10T18:50:17.620Z 28 560-579 7 7 7 9 9 9 F F F
12422023-12-10T18:50:17.620Z 29 580-599 5 5 5 7 7 7 F F F
12432023-12-10T18:50:17.620ZMax gen: 9, Max flush: 12
12442023-12-10T18:50:17.624Z
12452023-12-10T18:50:17.624ZBegin loop 5
12462023-12-10T18:50:17.624ZDownstairs to restart: 0
12472023-12-10T18:50:17.650Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.423187917Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199}
12482023-12-10T18:50:17.655Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.423596898Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199}
12492023-12-10T18:50:17.655Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.423673096Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199}
12502023-12-10T18:50:17.655Z{"msg":"Crucible 1f005949-330c-400d-be2e-923f203a22b9 has session id: f953d73e-fe14-469b-bdaa-ce1caef72d99","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.423715264Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12512023-12-10T18:50:17.655Z{Crucible runtime is spawned
12522023-12-10T18:50:17.655Z"msg":"Upstairs opts: Upstairs UUID: 1f005949-330c-400d-be2e-923f203a22b9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30The guest has requested activation with gen:10
12532023-12-10T18:50:17.656Z,"time":"2023-12-10T18:50:16.423910543Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12542023-12-10T18:50:17.656Z{"msg":"Crucible stats registered with UUID: 1f005949-330c-400d-be2e-923f203a22b9","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.424103079Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12552023-12-10T18:50:17.656Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.424166196Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"looper":"1","session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12562023-12-10T18:50:17.656Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.424326028Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"looper":"2","session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12572023-12-10T18:50:17.656Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.424376853Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"looper":"0","session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12582023-12-10T18:50:17.656Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.424417682Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99","task":"up_listen"}
12592023-12-10T18:50:17.656Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.424536442Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12602023-12-10T18:50:17.656Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.424643195Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12612023-12-10T18:50:17.656Z{"msg":"1f005949-330c-400d-be2e-923f203a22b9 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.424683172Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12622023-12-10T18:50:17.656Z{"msg":"Set desired generation to :10","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.424708447Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12632023-12-10T18:50:17.656Z{"msg":"[1] 1f005949-330c-400d-be2e-923f203a22b9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.424729799Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"looper":"1","session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12642023-12-10T18:50:17.656Z{"msg":"[2] 1f005949-330c-400d-be2e-923f203a22b9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.424751979Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"looper":"2","session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12652023-12-10T18:50:17.656Z{"msg":"[0] 1f005949-330c-400d-be2e-923f203a22b9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.424795724Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"looper":"0","session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12662023-12-10T18:50:17.656Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.424820254Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12672023-12-10T18:50:17.656Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.425109366Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12682023-12-10T18:50:17.656Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.425231978Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12692023-12-10T18:50:17.657Z{"msg":"[1] 1f005949-330c-400d-be2e-923f203a22b9 (f953d73e-fe14-469b-bdaa-ce1caef72d99) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.425526254Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12702023-12-10T18:50:17.657Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.425579563Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12712023-12-10T18:50:17.658Z{"msg":"[1] client is_active_req TRUE, promote! session f953d73e-fe14-469b-bdaa-ce1caef72d99","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.425613417Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12722023-12-10T18:50:17.658Z{"msg":"[2] 1f005949-330c-400d-be2e-923f203a22b9 (f953d73e-fe14-469b-bdaa-ce1caef72d99) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.425792185Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12732023-12-10T18:50:17.658Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.425907669Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12742023-12-10T18:50:17.658Z{"msg":"[2] client is_active_req TRUE, promote! session f953d73e-fe14-469b-bdaa-ce1caef72d99","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.42595209Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12752023-12-10T18:50:17.658Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.426428034Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12762023-12-10T18:50:17.658Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.426564803Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12772023-12-10T18:50:17.658Z{"msg":"1f005949-330c-400d-be2e-923f203a22b9 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.426652491Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12782023-12-10T18:50:17.658Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.426695068Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12792023-12-10T18:50:17.658Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.426769883Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12802023-12-10T18:50:17.658Z{"msg":"1f005949-330c-400d-be2e-923f203a22b9 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.426811028Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12812023-12-10T18:50:17.658Z{"msg":"[0] 1f005949-330c-400d-be2e-923f203a22b9 (f953d73e-fe14-469b-bdaa-ce1caef72d99) New WaitActive WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.42687846Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12822023-12-10T18:50:17.658Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.426910585Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12832023-12-10T18:50:17.658Z{"msg":"[0] client is_active_req TRUE, promote! session f953d73e-fe14-469b-bdaa-ce1caef72d99","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.426936018Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12842023-12-10T18:50:17.658Z{"msg":"[1] 1f005949-330c-400d-be2e-923f203a22b9 (f953d73e-fe14-469b-bdaa-ce1caef72d99) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.427262618Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12852023-12-10T18:50:17.658Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.427338166Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12862023-12-10T18:50:17.658Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:16.427378214Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12872023-12-10T18:50:17.658Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.427410369Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12882023-12-10T18:50:17.664Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.427512205Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12892023-12-10T18:50:17.665Z{"msg":"1f005949-330c-400d-be2e-923f203a22b9 WaitActive WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.42757459Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12902023-12-10T18:50:17.665Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.427600224Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12912023-12-10T18:50:17.665Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.427620475Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12922023-12-10T18:50:17.666Z{"msg":"[2] 1f005949-330c-400d-be2e-923f203a22b9 (f953d73e-fe14-469b-bdaa-ce1caef72d99) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.42768251Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12932023-12-10T18:50:17.666Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.427708107Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12942023-12-10T18:50:17.666Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:16.42775063Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12952023-12-10T18:50:17.666Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.427773576Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12962023-12-10T18:50:17.666Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.427795812Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12972023-12-10T18:50:17.666Z{"msg":"1f005949-330c-400d-be2e-923f203a22b9 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.427826349Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12982023-12-10T18:50:17.666Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.427898341Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
12992023-12-10T18:50:17.666Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.427941647Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13002023-12-10T18:50:17.667Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.428114067Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13012023-12-10T18:50:17.667Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.428176054Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13022023-12-10T18:50:17.667Z{"msg":"1f005949-330c-400d-be2e-923f203a22b9 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.428201066Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13032023-12-10T18:50:17.667Z{"msg":"[0] 1f005949-330c-400d-be2e-923f203a22b9 (f953d73e-fe14-469b-bdaa-ce1caef72d99) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.428833356Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13042023-12-10T18:50:17.667Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.428892361Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13052023-12-10T18:50:17.667Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:16.428981932Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13062023-12-10T18:50:17.667Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.429026654Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13072023-12-10T18:50:17.668Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.429050019Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13082023-12-10T18:50:17.668Z{"msg":"1f005949-330c-400d-be2e-923f203a22b9 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.42910152Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13092023-12-10T18:50:17.668Z{"msg":"[0]R flush_numbers[0..12]: [1, 11, 7, 7, 7, 9, 9, 2, 9, 8, 7, 11]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.429125667Z","The guest has finished waiting for activation with:10
13102023-12-10T18:50:17.669Zhostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"Wait for a query_work_queue command to finish before sending IO
13112023-12-10T18:50:17.669Zf953d73e-fe14-469b-bdaa-ce1caef72d99"}
13122023-12-10T18:50:17.669Z{"msg":"[0]R generation[0..12]: [1, 8, 5, 5, 5, 7, 7, 2, 7, 6, 5, 8]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.429231079Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13132023-12-10T18:50:17.669Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.429281441Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13142023-12-10T18:50:17.669Z{"msg":"[1]R flush_numbers[0..12]: [1, 11, 7, 7, 7, 9, 9, 2, 9, 8, 7, 11]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.429305516Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13152023-12-10T18:50:17.669Z{"msg":"[1]R generation[0..12]: [1, 8, 5, 5, 5, 7, 7, 2, 7, 6, 5, 8]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.429326156Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13162023-12-10T18:50:17.669Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.429348116Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13172023-12-10T18:50:17.669Z{"msg":"[2]R flush_numbers[0..12]: [1, 11, 7, 7, 7, 9, 9, 2, 9, 8, 7, 11]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.429369149Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13182023-12-10T18:50:17.669Z{"msg":"[2]R generation[0..12]: [1, 8, 5, 5, 5, 7, 7, 2, 7, 6, 5, 8]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.42938986Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13192023-12-10T18:50:17.669Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.42943306Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13202023-12-10T18:50:17.669Z{"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.429455255Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13212023-12-10T18:50:17.670Z{"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.42947526Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13222023-12-10T18:50:17.670Z{"msg":"Next flush: 13","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.429518765Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13232023-12-10T18:50:17.670Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.429540712Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13242023-12-10T18:50:17.671Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.429561453Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13252023-12-10T18:50:17.671Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.429601717Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13262023-12-10T18:50:17.671Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.429624525Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13272023-12-10T18:50:17.671Z{"msg":"1f005949-330c-400d-be2e-923f203a22b9 is now active with session: f953d73e-fe14-469b-bdaa-ce1caef72d99","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.429646149Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13282023-12-10T18:50:17.671Z{"msg":"1f005949-330c-400d-be2e-923f203a22b9 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.429668325Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13292023-12-10T18:50:17.671Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.429709001Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13302023-12-10T18:50:17.671Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.429730655Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13312023-12-10T18:50:17.671Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.429750552Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13322023-12-10T18:50:17.671Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.429790335Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13332023-12-10T18:50:17.671Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.429812395Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13342023-12-10T18:50:17.671Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.429832139Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13352023-12-10T18:50:17.671Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.429851555Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13362023-12-10T18:50:17.671Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.429871008Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13372023-12-10T18:50:17.671Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.429890457Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13382023-12-10T18:50:17.671Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.42991012Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13392023-12-10T18:50:17.672Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:16.42992946Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1199,"session_id":"f953d73e-fe14-469b-bdaa-ce1caef72d99"}
13402023-12-10T18:50:17.672ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
13412023-12-10T18:50:17.672ZLoading write count information from file "/tmp/test_repair_verify.data"
13422023-12-10T18:50:17.672ZRun Repair workload
13432023-12-10T18:50:17.672Z01/30 Flush
13442023-12-10T18:50:17.672Z02/30 Write block 352 len 8192 data: 3 3
13452023-12-10T18:50:17.672Z03/30 Write block 528 len 20480 data: 4 4 4 3 3
13462023-12-10T18:50:17.672Z04/30 Read block 124 len 8192
13472023-12-10T18:50:17.672Z05/30 Read block 365 len 16384
13482023-12-10T18:50:17.676Z06/30 Read block 372 len 16384
13492023-12-10T18:50:17.676Z07/30 Read block 108 len 4096
13502023-12-10T18:50:17.676Z08/30 Read block 362 len 40960
13512023-12-10T18:50:17.676Z09/30 Read block 182 len 40960
13522023-12-10T18:50:17.676Z10/30 Write block 195 len 12288 data: 2 2 2
13532023-12-10T18:50:17.676Z11/30 Flush
13542023-12-10T18:50:17.679Z12/30 Flush
13552023-12-10T18:50:17.679Z13/30 Write block 347 len 4096 data: 3
13562023-12-10T18:50:17.679Z14/30 Read block 460 len 12288
13572023-12-10T18:50:17.682Z15/30 Read block 108 len 32768
13582023-12-10T18:50:17.688Z16/30 Write block 440 len 32768 data: 3 3 2 2 2 2 2 2
13592023-12-10T18:50:17.688Z17/30 Write block 541 len 24576 data: 4 5 5 4 4 4
13602023-12-10T18:50:17.690Z18/30 Write block 85 len 20480 data: 2 3 3 3 3
13612023-12-10T18:50:17.691Z19/30 Write block 65 len 28672 data: 2 3 3 3 3 2 2
13622023-12-10T18:50:17.691Z20/30 Write block 309 len 4096 data: 2
13632023-12-10T18:50:17.691Z21/30 Flush
13642023-12-10T18:50:17.693Z22/30 Write block 418 len 28672 data: 2 2 2 2 2 2 3
13652023-12-10T18:50:17.694Z23/30 Read block 165 len 8192
13662023-12-10T18:50:17.696Z24/30 Read block 15 len 8192
13672023-12-10T18:50:17.696Z25/30 Write block 439 len 12288 data: 3 4 4
13682023-12-10T18:50:17.697Z26/30 Read block 158 len 28672
13692023-12-10T18:50:17.699Z27/30 Read block 199 len 16384
13702023-12-10T18:50:17.702Z28/30 Read block 175 len 40960
13712023-12-10T18:50:17.705Z29/30 Write block 343 len 28672 data: 2 2 2 2 4 3 3
13722023-12-10T18:50:17.705Z30/30 Read block 338 len 36864
13732023-12-10T18:50:17.709Z----------------------------------------------------------------
13742023-12-10T18:50:17.709Z Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:29
13752023-12-10T18:50:17.709ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
13762023-12-10T18:50:17.709Z 2 Acked 1001 Write 2 Done Done Done false
13772023-12-10T18:50:17.709Z 3 Acked 1002 Write 5 Done Done Done false
13782023-12-10T18:50:17.709Z 4 Acked 1003 Read 2 Done Done Done false
13792023-12-10T18:50:17.709Z 5 Acked 1004 Read 4 Done Done Done false
13802023-12-10T18:50:17.709Z 6 Acked 1005 Read 4 Done Done Done false
13812023-12-10T18:50:17.709Z 7 Acked 1006 Read 1 Sent Done Done false
13822023-12-10T18:50:17.709Z 8 Acked 1007 Read 10 Sent Done Done false
13832023-12-10T18:50:17.709Z 9 Acked 1008 Read 10 Sent Done Done false
13842023-12-10T18:50:17.709Z 10 Acked 1009 Write 3 Sent Done Done false
13852023-12-10T18:50:17.709Z 11 Acked 1010 Flush 0 Sent Done Done false
13862023-12-10T18:50:17.709Z 12 Acked 1011 Flush 0 Sent Done Done false
13872023-12-10T18:50:17.710Z 13 Acked 1012 Write 1 Sent Done Done false
13882023-12-10T18:50:17.710Z 14 Acked 1013 Read 3 Sent Done Done false
13892023-12-10T18:50:17.710Z 15 Acked 1014 Read 8 Sent Done Done false
13902023-12-10T18:50:17.710Z 16 Acked 1015 Write 8 Sent Done Done false
13912023-12-10T18:50:17.710Z 17 Acked 1016 Write 6 Sent Done Done false
13922023-12-10T18:50:17.710Z 18 Acked 1017 Write 5 Sent Done Done false
13932023-12-10T18:50:17.710Z 19 Acked 1018 Write 7 Sent Done Done false
13942023-12-10T18:50:17.710Z 20 Acked 1019 Write 1 Sent Done Done false
13952023-12-10T18:50:17.710Z 21 Acked 1020 Flush 0 Sent Done Done false
13962023-12-10T18:50:17.710Z 22 Acked 1021 Write 7 Sent Done Done false
13972023-12-10T18:50:17.710Z 23 Acked 1022 Read 2 Sent Done Done false
13982023-12-10T18:50:17.710Z 24 Acked 1023 Read 2 Sent Done Done false
13992023-12-10T18:50:17.710Z 25 Acked 1024 Write 3 Sent Done Done false
14002023-12-10T18:50:17.710Z 26 Acked 1025 Read 7 Sent Done Done false
14012023-12-10T18:50:17.710Z 27 Acked 1026 Read 4 Sent Done Done false
14022023-12-10T18:50:17.711Z 28 Acked 1027 Read 10 Sent Done Done false
14032023-12-10T18:50:17.711Z 29 Acked 1028 Write 7 Sent Done Done false
14042023-12-10T18:50:17.711Z 30 Acked 1029 Read 9 Sent Sent Done false
14052023-12-10T18:50:17.711Z STATES DS:0 DS:1 DS:2 TOTAL
14062023-12-10T18:50:17.711Z New 0 0 0 0
14072023-12-10T18:50:17.711Z Sent 24 1 0 25
14082023-12-10T18:50:17.711Z Done 5 28 29 62
14092023-12-10T18:50:17.711Z Skipped 0 0 0 0
14102023-12-10T18:50:17.711Z Error 0 0 0 0
14112023-12-10T18:50:17.711ZLast Flush: 1000 1020 1020
14122023-12-10T18:50:17.711ZDownstairs last five completed: 1000
14132023-12-10T18:50:17.711ZUpstairs last five completed: 30 29 28 27 26 25
14142023-12-10T18:50:17.714ZWrote out file "/tmp/test_repair_verify.data"
14152023-12-10T18:50:17.717Z
14162023-12-10T18:50:17.721ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
14172023-12-10T18:50:17.747ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14182023-12-10T18:50:17.747Z 0 000-019 1 1 1 1 1 1 F F F
14192023-12-10T18:50:17.750Z 1 020-039 8 8 8 11 11 11 F F F
14202023-12-10T18:50:17.750Z 2 040-059 5 5 5 7 7 7 F F F
14212023-12-10T18:50:17.751Z 3 060-079 5 10 10 7 16 16 F F F <---
14222023-12-10T18:50:17.751Z 4 080-099 5 10 10 7 16 16 F F F <---
14232023-12-10T18:50:17.751Z 5 100-119 7 7 7 9 9 9 F F F
14242023-12-10T18:50:17.751Z 6 120-139 7 7 7 9 9 9 F F F
14252023-12-10T18:50:17.751Z 7 140-159 2 2 2 2 2 2 F F F
14262023-12-10T18:50:17.751Z 8 160-179 7 7 7 9 9 9 F F F
14272023-12-10T18:50:17.752Z 9 180-199 6 10 10 8 14 14 F F F <---
14282023-12-10T18:50:17.752Z 10 200-219 5 5 5 7 7 7 F F F
14292023-12-10T18:50:17.752Z 11 220-239 8 8 8 11 11 11 F F F
14302023-12-10T18:50:17.752Z 12 240-259 7 7 7 9 9 9 F F F
14312023-12-10T18:50:17.752Z 13 260-279 8 8 8 11 11 11 F F F
14322023-12-10T18:50:17.752Z 14 280-299 7 7 7 9 9 9 F F F
14332023-12-10T18:50:17.752Z 15 300-319 1 10 10 1 16 16 F F F <---
14342023-12-10T18:50:17.752Z 16 320-339 5 5 5 7 7 7 F F F
14352023-12-10T18:50:17.753Z 17 340-359 9 10 10 12 16 16 T T T <---
14362023-12-10T18:50:17.753Z 18 360-379 9 9 9 12 12 12 F F F
14372023-12-10T18:50:17.753Z 19 380-399 9 9 9 12 12 12 F F F
14382023-12-10T18:50:17.753Z 20 400-419 1 1 1 1 1 1 F T T <---
14392023-12-10T18:50:17.753Z 21 420-439 9 9 9 12 12 12 F T T <---
14402023-12-10T18:50:17.753Z 22 440-459 3 10 10 3 16 16 F T T <---
14412023-12-10T18:50:17.754Z 23 460-479 1 1 1 1 1 1 F F F
14422023-12-10T18:50:17.754Z 24 480-499 9 9 9 12 12 12 F F F
14432023-12-10T18:50:17.754Z 25 500-519 9 9 9 12 12 12 F F F
14442023-12-10T18:50:17.754Z 26 520-539 9 10 10 12 14 14 T F F <---
14452023-12-10T18:50:17.754Z 27 540-559 9 10 10 12 16 16 F F F <---
14462023-12-10T18:50:17.755Z 28 560-579 7 7 7 9 9 9 F F F
14472023-12-10T18:50:17.755Z 29 580-599 5 5 5 7 7 7 F F F
14482023-12-10T18:50:17.755ZMax gen: 10, Max flush: 16
14492023-12-10T18:50:17.755ZError: Difference in extent metadata found!
14502023-12-10T18:50:17.761ZA Difference in extent metadata is expected here
14512023-12-10T18:50:17.762ZOn loop 5
14522023-12-10T18:50:17.762Z
14532023-12-10T18:50:17.762ZVerifying data now
14542023-12-10T18:50:24.527ZNo change after verify
14552023-12-10T18:50:24.527ZLoop: 5 Downstairs dump after verify (and repair):
14562023-12-10T18:50:24.733ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14572023-12-10T18:50:24.733Z 0 000-019 1 1 1 1 1 1 F F F
14582023-12-10T18:50:24.733Z 1 020-039 8 8 8 11 11 11 F F F
14592023-12-10T18:50:24.734Z 2 040-059 5 5 5 7 7 7 F F F
14602023-12-10T18:50:24.734Z 3 060-079 10 10 10 16 16 16 F F F
14612023-12-10T18:50:24.734Z 4 080-099 10 10 10 16 16 16 F F F
14622023-12-10T18:50:24.734Z 5 100-119 7 7 7 9 9 9 F F F
14632023-12-10T18:50:24.734Z 6 120-139 7 7 7 9 9 9 F F F
14642023-12-10T18:50:24.734Z 7 140-159 2 2 2 2 2 2 F F F
14652023-12-10T18:50:24.734Z 8 160-179 7 7 7 9 9 9 F F F
14662023-12-10T18:50:24.734Z 9 180-199 10 10 10 14 14 14 F F F
14672023-12-10T18:50:24.734Z 10 200-219 5 5 5 7 7 7 F F F
14682023-12-10T18:50:24.734Z 11 220-239 8 8 8 11 11 11 F F F
14692023-12-10T18:50:24.735Z 12 240-259 7 7 7 9 9 9 F F F
14702023-12-10T18:50:24.735Z 13 260-279 8 8 8 11 11 11 F F F
14712023-12-10T18:50:24.735Z 14 280-299 7 7 7 9 9 9 F F F
14722023-12-10T18:50:24.735Z 15 300-319 10 10 10 16 16 16 F F F
14732023-12-10T18:50:24.735Z 16 320-339 5 5 5 7 7 7 F F F
14742023-12-10T18:50:24.735Z 17 340-359 11 11 11 17 17 17 F F F
14752023-12-10T18:50:24.735Z 18 360-379 9 9 9 12 12 12 F F F
14762023-12-10T18:50:24.735Z 19 380-399 9 9 9 12 12 12 F F F
14772023-12-10T18:50:24.737Z 20 400-419 11 11 11 17 17 17 F F F
14782023-12-10T18:50:24.737Z 21 420-439 11 11 11 17 17 17 F F F
14792023-12-10T18:50:24.737Z 22 440-459 11 11 11 17 17 17 F F F
14802023-12-10T18:50:24.737Z 23 460-479 1 1 1 1 1 1 F F F
14812023-12-10T18:50:24.737Z 24 480-499 9 9 9 12 12 12 F F F
14822023-12-10T18:50:24.737Z 25 500-519 9 9 9 12 12 12 F F F
14832023-12-10T18:50:24.737Z 26 520-539 10 10 10 14 14 14 F F F
14842023-12-10T18:50:24.737Z 27 540-559 10 10 10 16 16 16 F F F
14852023-12-10T18:50:24.737Z 28 560-579 7 7 7 9 9 9 F F F
14862023-12-10T18:50:24.737Z 29 580-599 5 5 5 7 7 7 F F F
14872023-12-10T18:50:24.738ZMax gen: 11, Max flush: 17
14882023-12-10T18:50:24.738Z
14892023-12-10T18:50:24.738ZBegin loop 6
14902023-12-10T18:50:24.738ZDownstairs to restart: 1
14912023-12-10T18:50:24.738Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.632155407Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207}
14922023-12-10T18:50:24.738Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.632580104Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207}
14932023-12-10T18:50:24.738Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.632608197Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207}
14942023-12-10T18:50:24.738Z{"msg":"Crucible 8b83e9aa-5ee8-41a4-8617-c7571c6214bf has session id: f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.632629681Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
14952023-12-10T18:50:24.738Z{"msg":"Upstairs opts: Upstairs UUID: 8b83e9aa-5ee8-41a4-8617-c7571c6214bf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.632692619Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
14962023-12-10T18:50:24.739Z{"msg":"Crucible stats registered with UUID: 8b83e9aa-5ee8-41a4-8617-c7571c6214bf","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.63271923Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
14972023-12-10T18:50:24.739ZCrucible runtime is spawned
14982023-12-10T18:50:24.739ZThe guest has requested activation with gen:12
14992023-12-10T18:50:24.739Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.633078986Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"looper":"2","session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15002023-12-10T18:50:24.739Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.633186098Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"looper":"1","session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15012023-12-10T18:50:24.739Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.633229393Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"looper":"0","session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15022023-12-10T18:50:24.740Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.63326858Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24","task":"up_listen"}
15032023-12-10T18:50:24.740Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.633348201Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15042023-12-10T18:50:24.740Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.633389268Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15052023-12-10T18:50:24.740Z{"msg":"8b83e9aa-5ee8-41a4-8617-c7571c6214bf active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.633428977Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15062023-12-10T18:50:24.740Z{"msg":"Set desired generation to :12","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.633467135Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15072023-12-10T18:50:24.742Z{"msg":"[2] 8b83e9aa-5ee8-41a4-8617-c7571c6214bf looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.633579805Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"looper":"2","session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15082023-12-10T18:50:24.742Z{"msg":"[0] 8b83e9aa-5ee8-41a4-8617-c7571c6214bf looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.633625628Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"looper":"0","session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15092023-12-10T18:50:24.742Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.633756414Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15102023-12-10T18:50:24.742Z{"msg":"[1] 8b83e9aa-5ee8-41a4-8617-c7571c6214bf looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.633845135Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"looper":"1","session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15112023-12-10T18:50:24.742Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.634255818Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15122023-12-10T18:50:24.742Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.634321169Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15132023-12-10T18:50:24.742Z{"msg":"[2] 8b83e9aa-5ee8-41a4-8617-c7571c6214bf (f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.634474403Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15142023-12-10T18:50:24.743Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.634713423Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15152023-12-10T18:50:24.743Z{"msg":"[2] client is_active_req TRUE, promote! session f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.634800105Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15162023-12-10T18:50:24.743Z{"msg":"[0] 8b83e9aa-5ee8-41a4-8617-c7571c6214bf (f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.634921305Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15172023-12-10T18:50:24.743Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.634966633Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15182023-12-10T18:50:24.744Z{"msg":"[0] client is_active_req TRUE, promote! session f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.635001988Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15192023-12-10T18:50:24.744Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.635486295Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15202023-12-10T18:50:24.744Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.635591539Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15212023-12-10T18:50:24.744Z{"msg":"8b83e9aa-5ee8-41a4-8617-c7571c6214bf WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.63563721Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15222023-12-10T18:50:24.744Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.635719034Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15232023-12-10T18:50:24.744Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.635770506Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15242023-12-10T18:50:24.744Z{"msg":"8b83e9aa-5ee8-41a4-8617-c7571c6214bf WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.635809345Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15252023-12-10T18:50:24.744Z{"msg":"[2] 8b83e9aa-5ee8-41a4-8617-c7571c6214bf (f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.636325Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15262023-12-10T18:50:24.745Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.636400194Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15272023-12-10T18:50:24.745Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:21.636448939Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15282023-12-10T18:50:24.745Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.636472072Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15292023-12-10T18:50:24.745Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.63649286Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15302023-12-10T18:50:24.745Z{"msg":"8b83e9aa-5ee8-41a4-8617-c7571c6214bf WaitActive New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.636534315Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15312023-12-10T18:50:24.745Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.636557364Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15322023-12-10T18:50:24.745Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.636577342Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15332023-12-10T18:50:24.745Z{"msg":"[1] 8b83e9aa-5ee8-41a4-8617-c7571c6214bf (f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24) WaitActive New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.636610796Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15342023-12-10T18:50:24.746Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.636665832Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15352023-12-10T18:50:24.746Z{"msg":"[1] client is_active_req TRUE, promote! session f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.636707674Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15362023-12-10T18:50:24.747Z{"msg":"[0] 8b83e9aa-5ee8-41a4-8617-c7571c6214bf (f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.636769238Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15372023-12-10T18:50:24.747Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.636795395Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15382023-12-10T18:50:24.747Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:21.636821194Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15392023-12-10T18:50:24.747Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.636876077Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15402023-12-10T18:50:24.747Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.636901717Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15412023-12-10T18:50:24.747Z{"msg":"8b83e9aa-5ee8-41a4-8617-c7571c6214bf WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.63692422Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15422023-12-10T18:50:24.747Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.636960659Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15432023-12-10T18:50:24.747Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.636983832Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15442023-12-10T18:50:24.748Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.637346636Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15452023-12-10T18:50:24.748Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.637410934Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15462023-12-10T18:50:24.748Z{"msg":"8b83e9aa-5ee8-41a4-8617-c7571c6214bf WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.637512381Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15472023-12-10T18:50:24.749Z{"msg":"[1] 8b83e9aa-5ee8-41a4-8617-c7571c6214bf (f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.637966961Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15482023-12-10T18:50:24.749Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.638019034Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15492023-12-10T18:50:24.749Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:21.638051182Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15502023-12-10T18:50:24.749Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.638110511Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15512023-12-10T18:50:24.749Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.63813656Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15522023-12-10T18:50:24.749Z{"msg":"8b83e9aa-5ee8-41a4-8617-c7571c6214bf WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.638179315Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15532023-12-10T18:50:24.749Z{"msg":"[0]R flush_numbers[0..12]: [1, 11, 7, 16, 16, 9, 9, 2, 9, 14, 7, 11]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.638204937Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15542023-12-10T18:50:24.749Z{"msg":"[0]R generation[0..12]: [1, 8, 5, 10, 10, 7, 7, 2, 7, 10, 5, 8]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.638227491Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15552023-12-10T18:50:24.750Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v"The guest has finished waiting for activation with:12
15562023-12-10T18:50:24.877Z:0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
15572023-12-10T18:50:24.877Z,"time":"2023-12-10T18:50:21.638274685Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15582023-12-10T18:50:24.878Z{"msg":"[1]R flush_numbers[0..12]: [1, 11, 7, 16, 16, 9, 9, 2, 9, 14, 7, 11]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.6382991Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15592023-12-10T18:50:24.878Z{"msg":"[1]R generation[0..12]: [1, 8, 5, 10, 10, 7, 7, 2, 7, 10, 5, 8]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.638341603Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15602023-12-10T18:50:24.878Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.638366077Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15612023-12-10T18:50:24.878Z{"msg":"[2]R flush_numbers[0..12]: [1, 11, 7, 16, 16, 9, 9, 2, 9, 14, 7, 11]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.638386916Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15622023-12-10T18:50:24.878Z{"msg":"[2]R generation[0..12]: [1, 8, 5, 10, 10, 7, 7, 2, 7, 10, 5, 8]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.638408041Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15632023-12-10T18:50:24.878Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.638443987Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15642023-12-10T18:50:24.878Z{"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.638484061Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15652023-12-10T18:50:24.878Z{"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.638521553Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15662023-12-10T18:50:24.878Z{"msg":"Next flush: 18","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.63855575Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15672023-12-10T18:50:24.878Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.638639485Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15682023-12-10T18:50:24.879Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.638675997Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15692023-12-10T18:50:24.879Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.638740768Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15702023-12-10T18:50:24.879Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.638815487Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15712023-12-10T18:50:24.880Z{"msg":"8b83e9aa-5ee8-41a4-8617-c7571c6214bf is now active with session: f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.638895767Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15722023-12-10T18:50:24.880Z{"msg":"8b83e9aa-5ee8-41a4-8617-c7571c6214bf Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.638924874Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15732023-12-10T18:50:24.880Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.638976716Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15742023-12-10T18:50:24.880Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.639000764Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15752023-12-10T18:50:24.880Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.639049904Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15762023-12-10T18:50:24.880Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.6390902Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15772023-12-10T18:50:24.880Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.639125845Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15782023-12-10T18:50:24.880Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.639257429Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15792023-12-10T18:50:24.880Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
15802023-12-10T18:50:24.881Z,"time":"2023-12-10T18:50:21.639299882Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15812023-12-10T18:50:24.881Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.639338028Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15822023-12-10T18:50:24.881Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.63940669Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15832023-12-10T18:50:24.881Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.639446076Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15842023-12-10T18:50:24.881Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:21.639480733Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1207,"session_id":"f9f8dd07-1f14-4b4a-b100-c0b9a0f93f24"}
15852023-12-10T18:50:24.881ZLoading write count information from file "/tmp/test_repair_verify.data"
15862023-12-10T18:50:24.881ZRun Repair workload
15872023-12-10T18:50:24.881Z01/30 Write block 451 len 40960 data: 2 2 2 2 2 2 2 2 2 2
15882023-12-10T18:50:24.881Z02/30 Read block 286 len 24576
15892023-12-10T18:50:24.881Z03/30 Read block 548 len 40960
15902023-12-10T18:50:24.881Z04/30 Read block 574 len 28672
15912023-12-10T18:50:24.881Z05/30 Write block 357 len 8192 data: 3 2
15922023-12-10T18:50:24.882Z06/30 Write block 531 len 32768 data: 4 4 3 3 3 2 2 3
15932023-12-10T18:50:24.882Z07/30 Write block 21 len 20480 data: 2 3 3 3 3
15942023-12-10T18:50:24.882Z08/30 Read block 79 len 36864
15952023-12-10T18:50:24.882Z09/30 Flush
15962023-12-10T18:50:24.882Z10/30 Write block 540 len 12288 data: 4 5 6
15972023-12-10T18:50:24.883Z11/30 Read block 363 len 36864
15982023-12-10T18:50:24.883Z12/30 Write block 468 len 8192 data: 2 2
15992023-12-10T18:50:24.883Z13/30 Read block 344 len 32768
16002023-12-10T18:50:24.883Z14/30 Write block 193 len 40960 data: 2 2 3 3 3 2 2 2 2 2
16012023-12-10T18:50:24.884Z15/30 Read block 352 len 4096
16022023-12-10T18:50:24.884Z16/30 Write block 587 len 20480 data: 2 2 3 3 3
16032023-12-10T18:50:24.884Z17/30 Read block 438 len 32768
16042023-12-10T18:50:24.884Z18/30 Read block 486 len 16384
16052023-12-10T18:50:24.884Z19/30 Read block 500 len 40960
16062023-12-10T18:50:24.884Z20/30 Write block 574 len 32768 data: 2 2 2 2 2 2 2 2
16072023-12-10T18:50:24.884Z21/30 Read block 418 len 20480
16082023-12-10T18:50:24.884Z22/30 Write block 2 len 4096 data: 2
16092023-12-10T18:50:24.884Z23/30 Write block 197 len 12288 data: 4 3 3
16102023-12-10T18:50:24.884Z24/30 Write block 241 len 28672 data: 3 3 3 4 4 3 3
16112023-12-10T18:50:24.884Z25/30 Flush
16122023-12-10T18:50:24.885Z26/30 Write block 296 len 16384 data: 2 2 2 2
16132023-12-10T18:50:24.885Z27/30 Write block 461 len 28672 data: 2 2 2 2 2 2 2
16142023-12-10T18:50:24.885Z28/30 Read block 412 len 20480
16152023-12-10T18:50:24.885Z29/30 Read block 221 len 12288
16162023-12-10T18:50:24.885Z30/30 Write block 570 len 28672 data: 3 3 2 2 3 3 3
16172023-12-10T18:50:24.885Z----------------------------------------------------------------
16182023-12-10T18:50:24.885Z Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30
16192023-12-10T18:50:24.885ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
16202023-12-10T18:50:24.885Z 1 Acked 1000 Write 10 Done Done Done false
16212023-12-10T18:50:24.885Z 2 Acked 1001 Read 6 Done Done Done false
16222023-12-10T18:50:24.886Z 3 Acked 1002 Read 10 Done Sent Done false
16232023-12-10T18:50:24.886Z 4 Acked 1003 Read 7 Done Sent Done false
16242023-12-10T18:50:24.886Z 5 Acked 1004 Write 2 Done Sent Done false
16252023-12-10T18:50:24.886Z 6 Acked 1005 Write 8 Done Sent Done false
16262023-12-10T18:50:24.886Z 7 Acked 1006 Write 5 Done Sent Done false
16272023-12-10T18:50:24.886Z 8 Acked 1007 Read 9 Done Sent Done false
16282023-12-10T18:50:24.886Z 9 Acked 1008 Flush 0 Done Sent Done false
16292023-12-10T18:50:24.886Z 10 Acked 1009 Write 3 Done Sent Done false
16302023-12-10T18:50:24.886Z 11 Acked 1010 Read 9 Done Sent Done false
16312023-12-10T18:50:24.887Z 12 Acked 1011 Write 2 Done Sent Done false
16322023-12-10T18:50:24.887Z 13 Acked 1012 Read 8 Done Sent Done false
16332023-12-10T18:50:24.888Z 14 Acked 1013 Write 10 Done Sent Done false
16342023-12-10T18:50:24.888Z 15 Acked 1014 Read 1 Done Sent Done false
16352023-12-10T18:50:24.888Z 16 Acked 1015 Write 5 Done Sent Done false
16362023-12-10T18:50:24.888Z 17 Acked 1016 Read 8 Done Sent Done false
16372023-12-10T18:50:24.888Z 18 Acked 1017 Read 4 Done Sent Done false
16382023-12-10T18:50:24.888Z 19 Acked 1018 Read 10 Done Sent Done false
16392023-12-10T18:50:24.888Z 20 Acked 1019 Write 8 Done Sent Done false
16402023-12-10T18:50:24.888Z 21 Acked 1020 Read 5 Done Sent Done false
16412023-12-10T18:50:24.889Z 22 Acked 1021 Write 1 Done Sent Done false
16422023-12-10T18:50:24.889Z 23 Acked 1022 Write 3 Done Sent Done false
16432023-12-10T18:50:24.889Z 24 Acked 1023 Write 7 Done Sent Done false
16442023-12-10T18:50:24.889Z 25 Acked 1024 Flush 0 Done Sent Done false
16452023-12-10T18:50:24.889Z 26 Acked 1025 Write 4 Done Sent Done false
16462023-12-10T18:50:24.889Z 27 Acked 1026 Write 7 Done Sent Done false
16472023-12-10T18:50:24.889Z 28 Acked 1027 Read 5 Done Sent Done false
16482023-12-10T18:50:24.889Z 29 Acked 1028 Read 3 Done Sent Done false
16492023-12-10T18:50:24.889Z 30 Acked 1029 Write 7 Sent Sent Sent false
16502023-12-10T18:50:24.890Z STATES DS:0 DS:1 DS:2 TOTAL
16512023-12-10T18:50:24.890Z New 0 0 0 0
16522023-12-10T18:50:24.890Z Sent 1 28 1 30
16532023-12-10T18:50:24.890Z Done 29 2 29 60
16542023-12-10T18:50:24.890Z Skipped 0 0 0 0
16552023-12-10T18:50:24.890Z Error 0 0 0 0
16562023-12-10T18:50:24.931ZLast Flush: 1024 0 1024
16572023-12-10T18:50:24.931ZDownstairs last five completed:
16582023-12-10T18:50:24.931ZUpstairs last five completed: 30 29 28 27 26 25
16592023-12-10T18:50:24.931ZWrote out file "/tmp/test_repair_verify.data"
16602023-12-10T18:50:24.932Z
16612023-12-10T18:50:24.932ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
16622023-12-10T18:50:24.932ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
16632023-12-10T18:50:24.932Z 0 000-019 12 1 12 19 1 19 F F F <---
16642023-12-10T18:50:24.932Z 1 020-039 12 8 12 18 11 18 F F F <---
16652023-12-10T18:50:24.932Z 2 040-059 5 5 5 7 7 7 F F F
16662023-12-10T18:50:24.932Z 3 060-079 10 10 10 16 16 16 F F F
16672023-12-10T18:50:24.932Z 4 080-099 10 10 10 16 16 16 F F F
16682023-12-10T18:50:24.933Z 5 100-119 7 7 7 9 9 9 F F F
16692023-12-10T18:50:24.933Z 6 120-139 7 7 7 9 9 9 F F F
16702023-12-10T18:50:24.933Z 7 140-159 2 2 2 2 2 2 F F F
16712023-12-10T18:50:24.933Z 8 160-179 7 7 7 9 9 9 F F F
16722023-12-10T18:50:24.933Z 9 180-199 12 10 12 19 14 19 F F F <---
16732023-12-10T18:50:24.933Z 10 200-219 12 5 12 19 7 19 F F F <---
16742023-12-10T18:50:24.933Z 11 220-239 8 8 8 11 11 11 F F F
16752023-12-10T18:50:24.933Z 12 240-259 12 7 12 19 9 19 F F F <---
16762023-12-10T18:50:24.934Z 13 260-279 8 8 8 11 11 11 F F F
16772023-12-10T18:50:24.934Z 14 280-299 7 7 7 9 9 9 T F T <---
16782023-12-10T18:50:24.934Z 15 300-319 10 10 10 16 16 16 F F F
16792023-12-10T18:50:24.934Z 16 320-339 5 5 5 7 7 7 F F F
16802023-12-10T18:50:24.934Z 17 340-359 12 11 12 18 17 18 F F F <---
16812023-12-10T18:50:24.934Z 18 360-379 9 9 9 12 12 12 F F F
16822023-12-10T18:50:24.935Z 19 380-399 9 9 9 12 12 12 F F F
16832023-12-10T18:50:24.935Z 20 400-419 11 11 11 17 17 17 F F F
16842023-12-10T18:50:24.935Z 21 420-439 11 11 11 17 17 17 F F F
16852023-12-10T18:50:24.935Z 22 440-459 12 11 12 18 17 18 F T F <---
16862023-12-10T18:50:24.936Z 23 460-479 12 1 12 19 1 19 T T T <---
16872023-12-10T18:50:24.936Z 24 480-499 9 9 9 12 12 12 F F F
16882023-12-10T18:50:24.936Z 25 500-519 9 9 9 12 12 12 F F F
16892023-12-10T18:50:24.936Z 26 520-539 12 10 12 18 14 18 F F F <---
16902023-12-10T18:50:24.937Z 27 540-559 12 10 12 19 16 19 F F F <---
16912023-12-10T18:50:24.937Z 28 560-579 12 7 12 19 9 19 T F T <---
16922023-12-10T18:50:24.937Z 29 580-599 12 5 12 19 7 19 F F F <---
16932023-12-10T18:50:24.937ZMax gen: 12, Max flush: 19
16942023-12-10T18:50:24.937ZError: Difference in extent metadata found!
16952023-12-10T18:50:24.938ZA Difference in extent metadata is expected here
16962023-12-10T18:50:24.938ZOn loop 6
16972023-12-10T18:50:24.938Z
16982023-12-10T18:50:24.938ZVerifying data now
16992023-12-10T18:50:28.220ZNo change after verify
17002023-12-10T18:50:28.220ZLoop: 6 Downstairs dump after verify (and repair):
17012023-12-10T18:50:28.242ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
17022023-12-10T18:50:28.242Z 0 000-019 12 12 12 19 19 19 F F F
17032023-12-10T18:50:28.246Z 1 020-039 12 12 12 18 18 18 F F F
17042023-12-10T18:50:28.246Z 2 040-059 5 5 5 7 7 7 F F F
17052023-12-10T18:50:28.246Z 3 060-079 10 10 10 16 16 16 F F F
17062023-12-10T18:50:28.246Z 4 080-099 10 10 10 16 16 16 F F F
17072023-12-10T18:50:28.246Z 5 100-119 7 7 7 9 9 9 F F F
17082023-12-10T18:50:28.247Z 6 120-139 7 7 7 9 9 9 F F F
17092023-12-10T18:50:28.247Z 7 140-159 2 2 2 2 2 2 F F F
17102023-12-10T18:50:28.247Z 8 160-179 7 7 7 9 9 9 F F F
17112023-12-10T18:50:28.247Z 9 180-199 12 12 12 19 19 19 F F F
17122023-12-10T18:50:28.247Z 10 200-219 12 12 12 19 19 19 F F F
17132023-12-10T18:50:28.247Z 11 220-239 8 8 8 11 11 11 F F F
17142023-12-10T18:50:28.247Z 12 240-259 12 12 12 19 19 19 F F F
17152023-12-10T18:50:28.247Z 13 260-279 8 8 8 11 11 11 F F F
17162023-12-10T18:50:28.247Z 14 280-299 13 13 13 20 20 20 F F F
17172023-12-10T18:50:28.248Z 15 300-319 10 10 10 16 16 16 F F F
17182023-12-10T18:50:28.248Z 16 320-339 5 5 5 7 7 7 F F F
17192023-12-10T18:50:28.248Z 17 340-359 12 12 12 18 18 18 F F F
17202023-12-10T18:50:28.248Z 18 360-379 9 9 9 12 12 12 F F F
17212023-12-10T18:50:28.248Z 19 380-399 9 9 9 12 12 12 F F F
17222023-12-10T18:50:28.248Z 20 400-419 11 11 11 17 17 17 F F F
17232023-12-10T18:50:28.248Z 21 420-439 11 11 11 17 17 17 F F F
17242023-12-10T18:50:28.248Z 22 440-459 12 12 12 18 18 18 F F F
17252023-12-10T18:50:28.248Z 23 460-479 13 13 13 20 20 20 F F F
17262023-12-10T18:50:28.248Z 24 480-499 9 9 9 12 12 12 F F F
17272023-12-10T18:50:28.249Z 25 500-519 9 9 9 12 12 12 F F F
17282023-12-10T18:50:28.249Z 26 520-539 12 12 12 18 18 18 F F F
17292023-12-10T18:50:28.249Z 27 540-559 12 12 12 19 19 19 F F F
17302023-12-10T18:50:28.249Z 28 560-579 13 13 13 20 20 20 F F F
17312023-12-10T18:50:28.249Z 29 580-599 12 12 12 19 19 19 F F F
17322023-12-10T18:50:28.249ZMax gen: 13, Max flush: 20
17332023-12-10T18:50:28.253Z
17342023-12-10T18:50:28.253ZBegin loop 7
17352023-12-10T18:50:28.254ZDownstairs to restart: 0
17362023-12-10T18:50:28.280Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.050785811Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215}
17372023-12-10T18:50:28.280Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.051185829Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215}
17382023-12-10T18:50:28.289Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.051213682Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215}
17392023-12-10T18:50:28.289Z{"msg":"Crucible 04dfd63f-728d-4566-9610-df3d687be800 has session id: db948938-d4f7-416d-bf72-24c86bd207aa","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.051267401Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17402023-12-10T18:50:28.289Z{"msg":"Upstairs opts: Upstairs UUID: 04dfd63f-728d-4566-9610-df3d687be800, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.051313106Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17412023-12-10T18:50:28.289Z{"msg":"Crucible stats registered with UUID: 04dfd63f-728d-4566-9610-df3d687be800","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.051430904Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17422023-12-10T18:50:28.289Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.051636701Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"looper":"Crucible runtime is spawned
17432023-12-10T18:50:28.289Z0","session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17442023-12-10T18:50:28.289Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.05170966Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"looper":"1","session_id":"The guest has requested activation with gen:14
17452023-12-10T18:50:28.289Zdb948938-d4f7-416d-bf72-24c86bd207aa"}
17462023-12-10T18:50:28.289Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.051783372Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"looper":"2","session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17472023-12-10T18:50:28.290Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.051919318Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa","task":"up_listen"}
17482023-12-10T18:50:28.290Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.051966132Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17492023-12-10T18:50:28.291Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.05206988Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17502023-12-10T18:50:28.291Z{"msg":"[0] 04dfd63f-728d-4566-9610-df3d687be800 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.052206084Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"looper":"0","session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17512023-12-10T18:50:28.291Z{"msg":"[2] 04dfd63f-728d-4566-9610-df3d687be800 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.052348385Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"looper":"2","session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17522023-12-10T18:50:28.291Z{"msg":"[1] 04dfd63f-728d-4566-9610-df3d687be800 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.052391633Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"looper":"1","session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17532023-12-10T18:50:28.291Z{"msg":"04dfd63f-728d-4566-9610-df3d687be800 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.052430401Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17542023-12-10T18:50:28.291Z{"msg":"Set desired generation to :14","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.052489723Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17552023-12-10T18:50:28.291Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.052548881Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17562023-12-10T18:50:28.291Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.052849068Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17572023-12-10T18:50:28.291Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.052934707Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17582023-12-10T18:50:28.291Z{"msg":"[2] 04dfd63f-728d-4566-9610-df3d687be800 (db948938-d4f7-416d-bf72-24c86bd207aa) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.053263531Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17592023-12-10T18:50:28.293Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.053360695Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17602023-12-10T18:50:28.293Z{"msg":"[2] client is_active_req TRUE, promote! session db948938-d4f7-416d-bf72-24c86bd207aa","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.053586492Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17612023-12-10T18:50:28.293Z{"msg":"[1] 04dfd63f-728d-4566-9610-df3d687be800 (db948938-d4f7-416d-bf72-24c86bd207aa) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.053662175Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17622023-12-10T18:50:28.293Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.053718678Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17632023-12-10T18:50:28.294Z{"msg":"[1] client is_active_req TRUE, promote! session db948938-d4f7-416d-bf72-24c86bd207aa","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.053760694Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17642023-12-10T18:50:28.294Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.053935908Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17652023-12-10T18:50:28.294Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.054002891Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17662023-12-10T18:50:28.294Z{"msg":"04dfd63f-728d-4566-9610-df3d687be800 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.054033349Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17672023-12-10T18:50:28.294Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.054372919Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17682023-12-10T18:50:28.294Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.054455194Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17692023-12-10T18:50:28.294Z{"msg":"04dfd63f-728d-4566-9610-df3d687be800 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.054625384Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17702023-12-10T18:50:28.294Z{"msg":"[0] 04dfd63f-728d-4566-9610-df3d687be800 (db948938-d4f7-416d-bf72-24c86bd207aa) New WaitActive WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.054747294Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17712023-12-10T18:50:28.294Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.054816472Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17722023-12-10T18:50:28.295Z{"msg":"[0] client is_active_req TRUE, promote! session db948938-d4f7-416d-bf72-24c86bd207aa","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.054857925Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17732023-12-10T18:50:28.295Z{"msg":"[2] 04dfd63f-728d-4566-9610-df3d687be800 (db948938-d4f7-416d-bf72-24c86bd207aa) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.054994176Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17742023-12-10T18:50:28.295Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.055103481Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17752023-12-10T18:50:28.295Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:27.055143526Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17762023-12-10T18:50:28.295Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.055199424Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17772023-12-10T18:50:28.295Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.055239078Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17782023-12-10T18:50:28.296Z{"msg":"04dfd63f-728d-4566-9610-df3d687be800 WaitActive WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.055295957Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17792023-12-10T18:50:28.296Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.055335194Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17802023-12-10T18:50:28.296Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.055389128Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17812023-12-10T18:50:28.296Z{"msg":"[1] 04dfd63f-728d-4566-9610-df3d687be800 (db948938-d4f7-416d-bf72-24c86bd207aa) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.055432037Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17822023-12-10T18:50:28.296Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.055487079Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17832023-12-10T18:50:28.296Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:27.055525534Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17842023-12-10T18:50:28.297Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.05556036Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17852023-12-10T18:50:28.297Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.055597477Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17862023-12-10T18:50:28.297Z{"msg":"04dfd63f-728d-4566-9610-df3d687be800 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.055650334Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17872023-12-10T18:50:28.297Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.055692382Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17882023-12-10T18:50:28.297Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.055745822Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17892023-12-10T18:50:28.297Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.055786149Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17902023-12-10T18:50:28.297Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.055847961Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17912023-12-10T18:50:28.297Z{"msg":"04dfd63f-728d-4566-9610-df3d687be800 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.055888606Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17922023-12-10T18:50:28.297Z{"msg":"[0] 04dfd63f-728d-4566-9610-df3d687be800 (db948938-d4f7-416d-bf72-24c86bd207aa) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.05619837Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17932023-12-10T18:50:28.298Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.056286586Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17942023-12-10T18:50:28.298Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:27.056362217Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17952023-12-10T18:50:28.298Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.056431769Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17962023-12-10T18:50:28.298Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.056470562Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17972023-12-10T18:50:28.298Z{"msg":"04dfd63f-728d-4566-9610-df3d687be800 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.056539852Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
17982023-12-10T18:50:28.298Z{"msg":"[0]R flush_numbers[0..12]: [19, 18, 7, 16, 16, 9, 9, 2, 9, 19, 19, 11]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:14
17992023-12-10T18:50:28.300Z,"time":"2023-12-10T18:50:27.056582902Z","hostname"Wait for a query_work_queue command to finish before sending IO
18002023-12-10T18:50:28.300Z:"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18012023-12-10T18:50:28.300Z{"msg":"[0]R generation[0..12]: [12, 12, 5, 10, 10, 7, 7, 2, 7, 12, 12, 8]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.056627407Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18022023-12-10T18:50:28.301Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.056701404Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18032023-12-10T18:50:28.301Z{"msg":"[1]R flush_numbers[0..12]: [19, 18, 7, 16, 16, 9, 9, 2, 9, 19, 19, 11]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.056741305Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18042023-12-10T18:50:28.301Z{"msg":"[1]R generation[0..12]: [12, 12, 5, 10, 10, 7, 7, 2, 7, 12, 12, 8]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.05681364Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18052023-12-10T18:50:28.301Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.056857159Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18062023-12-10T18:50:28.301Z{"msg":"[2]R flush_numbers[0..12]: [19, 18, 7, 16, 16, 9, 9, 2, 9, 19, 19, 11]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.056922782Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18072023-12-10T18:50:28.301Z{"msg":"[2]R generation[0..12]: [12, 12, 5, 10, 10, 7, 7, 2, 7, 12, 12, 8]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.056963053Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18082023-12-10T18:50:28.301Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.056990626Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18092023-12-10T18:50:28.301Z{"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.057058858Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18102023-12-10T18:50:28.303Z{"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.057099927Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18112023-12-10T18:50:28.303Z{"msg":"Next flush: 21","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.057166935Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18122023-12-10T18:50:28.303Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.05725666Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18132023-12-10T18:50:28.303Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.057299662Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18142023-12-10T18:50:28.303Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.057374602Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18152023-12-10T18:50:28.303Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.057401114Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18162023-12-10T18:50:28.303Z{"msg":"04dfd63f-728d-4566-9610-df3d687be800 is now active with session: db948938-d4f7-416d-bf72-24c86bd207aa","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.057451434Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18172023-12-10T18:50:28.303Z{"msg":"04dfd63f-728d-4566-9610-df3d687be800 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.057474551Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18182023-12-10T18:50:28.303Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.057529956Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18192023-12-10T18:50:28.303Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.057584978Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18202023-12-10T18:50:28.304Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.057644782Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18212023-12-10T18:50:28.305Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.057676186Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18222023-12-10T18:50:28.305Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
18232023-12-10T18:50:28.305Z,"time":"2023-12-10T18:50:27.057737804Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18242023-12-10T18:50:28.305Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.057858025Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18252023-12-10T18:50:28.306Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.05788431Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18262023-12-10T18:50:28.306Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.057925708Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18272023-12-10T18:50:28.306Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.05794802Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18282023-12-10T18:50:28.306Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.057987963Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18292023-12-10T18:50:28.306Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:27.058020272Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1215,"session_id":"db948938-d4f7-416d-bf72-24c86bd207aa"}
18302023-12-10T18:50:28.306ZLoading write count information from file "/tmp/test_repair_verify.data"
18312023-12-10T18:50:28.306ZRun Repair workload
18322023-12-10T18:50:28.306Z01/30 Write block 387 len 8192 data: 3 3
18332023-12-10T18:50:28.311Z02/30 Flush
18342023-12-10T18:50:28.311Z03/30 Write block 282 len 24576 data: 3 2 3 4 4 4
18352023-12-10T18:50:28.311Z04/30 Write block 524 len 12288 data: 3 4 5
18362023-12-10T18:50:28.311Z05/30 Read block 322 len 12288
18372023-12-10T18:50:28.312Z06/30 Read block 225 len 40960
18382023-12-10T18:50:28.312Z07/30 Read block 261 len 8192
18392023-12-10T18:50:28.312Z08/30 Write block 322 len 4096 data: 2
18402023-12-10T18:50:28.312Z09/30 Read block 70 len 24576
18412023-12-10T18:50:28.312Z10/30 Write block 508 len 40960 data: 3 3 3 3 2 3 3 3 3 3
18422023-12-10T18:50:28.312Z11/30 Write block 467 len 40960 data: 3 3 3 2 2 2 2 2 2 2
18432023-12-10T18:50:28.312Z12/30 Write block 115 len 24576 data: 2 3 3 4 4 4
18442023-12-10T18:50:28.312Z13/30 Read block 46 len 28672
18452023-12-10T18:50:28.312Z14/30 Read block 351 len 40960
18462023-12-10T18:50:28.312Z15/30 Write block 17 len 4096 data: 2
18472023-12-10T18:50:28.312Z16/30 Flush
18482023-12-10T18:50:28.317Z17/30 Write block 422 len 4096 data: 3
18492023-12-10T18:50:28.317Z18/30 Read block 261 len 36864
18502023-12-10T18:50:28.322Z19/30 Read block 23 len 4096
18512023-12-10T18:50:28.322Z20/30 Flush
18522023-12-10T18:50:28.325Z21/30 Flush
18532023-12-10T18:50:28.325Z22/30 Write block 392 len 36864 data: 4 4 4 4 4 3 3 3 2
18542023-12-10T18:50:28.325Z23/30 Write block 546 len 12288 data: 5 3 3
18552023-12-10T18:50:28.325Z24/30 Read block 359 len 4096
18562023-12-10T18:50:28.329Z25/30 Flush
18572023-12-10T18:50:28.329Z26/30 Write block 148 len 20480 data: 2 2 2 2 2
18582023-12-10T18:50:28.329Z27/30 Read block 523 len 20480
18592023-12-10T18:50:28.332Z28/30 Read block 129 len 24576
18602023-12-10T18:50:28.335Z29/30 Write block 161 len 28672 data: 3 3 3 2 2 2 2
18612023-12-10T18:50:28.336Z30/30 Write block 471 len 36864 data: 3 3 3 3 3 3 2 2 2
18622023-12-10T18:50:28.339Z----------------------------------------------------------------
18632023-12-10T18:50:28.339Z Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:28
18642023-12-10T18:50:28.340ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
18652023-12-10T18:50:28.340Z 3 Acked 1002 Write 6 Done Done Done false
18662023-12-10T18:50:28.340Z 4 Acked 1003 Write 3 Done Done Done false
18672023-12-10T18:50:28.340Z 5 Acked 1004 Read 3 Done Done Done false
18682023-12-10T18:50:28.341Z 6 Acked 1005 Read 10 Sent Done Done false
18692023-12-10T18:50:28.341Z 7 Acked 1006 Read 2 Sent Done Done false
18702023-12-10T18:50:28.341Z 8 Acked 1007 Write 1 Sent Done Done false
18712023-12-10T18:50:28.341Z 9 Acked 1008 Read 6 Sent Done Done false
18722023-12-10T18:50:28.341Z 10 Acked 1009 Write 10 Sent Done Done false
18732023-12-10T18:50:28.341Z 11 Acked 1010 Write 10 Sent Done Done false
18742023-12-10T18:50:28.342Z 12 Acked 1011 Write 6 Sent Done Done false
18752023-12-10T18:50:28.342Z 13 Acked 1012 Read 7 Sent Done Done false
18762023-12-10T18:50:28.342Z 14 Acked 1013 Read 10 Sent Done Done false
18772023-12-10T18:50:28.342Z 15 Acked 1014 Write 1 Sent Done Done false
18782023-12-10T18:50:28.342Z 16 Acked 1015 Flush 0 Sent Done Done false
18792023-12-10T18:50:28.342Z 17 Acked 1016 Write 1 Sent Done Done false
18802023-12-10T18:50:28.342Z 18 Acked 1017 Read 9 Sent Done Done false
18812023-12-10T18:50:28.342Z 19 Acked 1018 Read 1 Sent Done Done false
18822023-12-10T18:50:28.343Z 20 Acked 1019 Flush 0 Sent Done Done false
18832023-12-10T18:50:28.343Z 21 Acked 1020 Flush 0 Sent Done Done false
18842023-12-10T18:50:28.343Z 22 Acked 1021 Write 9 Sent Done Done false
18852023-12-10T18:50:28.343Z 23 Acked 1022 Write 3 Sent Done Done false
18862023-12-10T18:50:28.343Z 24 Acked 1023 Read 1 Sent Done Done false
18872023-12-10T18:50:28.343Z 25 Acked 1024 Flush 0 Sent Done Done false
18882023-12-10T18:50:28.343Z 26 Acked 1025 Write 5 Sent Done Done false
18892023-12-10T18:50:28.343Z 27 Acked 1026 Read 5 Sent Done Done false
18902023-12-10T18:50:28.343Z 28 Acked 1027 Read 6 Sent Done Done false
18912023-12-10T18:50:28.343Z 29 Acked 1028 Write 7 Sent Done Done false
18922023-12-10T18:50:28.344Z 30 Acked 1029 Write 9 Sent Sent Sent false
18932023-12-10T18:50:28.344Z STATES DS:0 DS:1 DS:2 TOTAL
18942023-12-10T18:50:28.344Z New 0 0 0 0
18952023-12-10T18:50:28.344Z Sent 25 1 1 27
18962023-12-10T18:50:28.344Z Done 3 27 27 57
18972023-12-10T18:50:28.345Z Skipped 0 0 0 0
18982023-12-10T18:50:28.345Z Error 0 0 0 0
18992023-12-10T18:50:28.345ZLast Flush: 1001 1024 1024
19002023-12-10T18:50:28.346ZDownstairs last five completed: 1001 1000
19012023-12-10T18:50:28.346ZUpstairs last five completed: 30 29 28 27 26 25
19022023-12-10T18:50:28.346ZWrote out file "/tmp/test_repair_verify.data"
19032023-12-10T18:50:28.366Z
19042023-12-10T18:50:28.366ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
19052023-12-10T18:50:28.377ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
19062023-12-10T18:50:28.378Z 0 000-019 12 14 14 19 22 22 F F F <---
19072023-12-10T18:50:28.378Z 1 020-039 12 12 12 18 18 18 F F F
19082023-12-10T18:50:28.378Z 2 040-059 5 5 5 7 7 7 F F F
19092023-12-10T18:50:28.378Z 3 060-079 10 10 10 16 16 16 F F F
19102023-12-10T18:50:28.378Z 4 080-099 10 10 10 16 16 16 F F F
19112023-12-10T18:50:28.378Z 5 100-119 7 14 14 9 22 22 F F F <---
19122023-12-10T18:50:28.378Z 6 120-139 7 14 14 9 22 22 F F F <---
19132023-12-10T18:50:28.378Z 7 140-159 2 2 2 2 2 2 F T T <---
19142023-12-10T18:50:28.378Z 8 160-179 7 7 7 9 9 9 F T T <---
19152023-12-10T18:50:28.378Z 9 180-199 12 12 12 19 19 19 F F F
19162023-12-10T18:50:28.378Z 10 200-219 12 12 12 19 19 19 F F F
19172023-12-10T18:50:28.379Z 11 220-239 8 8 8 11 11 11 F F F
19182023-12-10T18:50:28.379Z 12 240-259 12 12 12 19 19 19 F F F
19192023-12-10T18:50:28.379Z 13 260-279 8 8 8 11 11 11 F F F
19202023-12-10T18:50:28.379Z 14 280-299 13 14 14 20 22 22 T F F <---
19212023-12-10T18:50:28.379Z 15 300-319 10 10 10 16 16 16 F F F
19222023-12-10T18:50:28.380Z 16 320-339 5 14 14 7 22 22 F F F <---
19232023-12-10T18:50:28.381Z 17 340-359 12 12 12 18 18 18 F F F
19242023-12-10T18:50:28.381Z 18 360-379 9 9 9 12 12 12 F F F
19252023-12-10T18:50:28.381Z 19 380-399 14 14 14 21 25 25 F F F <---
19262023-12-10T18:50:28.381Z 20 400-419 11 14 14 17 25 25 F F F <---
19272023-12-10T18:50:28.381Z 21 420-439 11 14 14 17 23 23 F F F <---
19282023-12-10T18:50:28.381Z 22 440-459 12 12 12 18 18 18 F F F
19292023-12-10T18:50:28.381Z 23 460-479 13 14 14 20 22 22 F T T <---
19302023-12-10T18:50:28.382Z 24 480-499 9 9 9 12 12 12 F F F
19312023-12-10T18:50:28.382Z 25 500-519 9 14 14 12 22 22 F F F <---
19322023-12-10T18:50:28.382Z 26 520-539 12 14 14 18 22 22 T F F <---
19332023-12-10T18:50:28.382Z 27 540-559 12 14 14 19 25 25 F F F <---
19342023-12-10T18:50:28.382Z 28 560-579 13 13 13 20 20 20 F F F
19352023-12-10T18:50:28.382Z 29 580-599 12 12 12 19 19 19 F F F
19362023-12-10T18:50:28.382ZMax gen: 14, Max flush: 25
19372023-12-10T18:50:28.382ZError: Difference in extent metadata found!
19382023-12-10T18:50:28.386ZA Difference in extent metadata is expected here
19392023-12-10T18:50:28.386ZOn loop 7
19402023-12-10T18:50:28.386Z
19412023-12-10T18:50:28.386ZVerifying data now
19422023-12-10T18:50:34.717ZNo change after verify
19432023-12-10T18:50:34.718ZLoop: 7 Downstairs dump after verify (and repair):
19442023-12-10T18:50:34.792ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
19452023-12-10T18:50:34.793Z 0 000-019 14 14 14 22 22 22 F F F
19462023-12-10T18:50:34.814Z 1 020-039 12 12 12 18 18 18 F F F
19472023-12-10T18:50:34.814Z 2 040-059 5 5 5 7 7 7 F F F
19482023-12-10T18:50:34.814Z 3 060-079 10 10 10 16 16 16 F F F
19492023-12-10T18:50:34.814Z 4 080-099 10 10 10 16 16 16 F F F
19502023-12-10T18:50:34.814Z 5 100-119 14 14 14 22 22 22 F F F
19512023-12-10T18:50:34.814Z 6 120-139 14 14 14 22 22 22 F F F
19522023-12-10T18:50:34.814Z 7 140-159 15 15 15 26 26 26 F F F
19532023-12-10T18:50:34.814Z 8 160-179 15 15 15 26 26 26 F F F
19542023-12-10T18:50:34.814Z 9 180-199 12 12 12 19 19 19 F F F
19552023-12-10T18:50:34.814Z 10 200-219 12 12 12 19 19 19 F F F
19562023-12-10T18:50:34.814Z 11 220-239 8 8 8 11 11 11 F F F
19572023-12-10T18:50:34.815Z 12 240-259 12 12 12 19 19 19 F F F
19582023-12-10T18:50:34.815Z 13 260-279 8 8 8 11 11 11 F F F
19592023-12-10T18:50:34.815Z 14 280-299 14 14 14 22 22 22 F F F
19602023-12-10T18:50:34.815Z 15 300-319 10 10 10 16 16 16 F F F
19612023-12-10T18:50:34.815Z 16 320-339 14 14 14 22 22 22 F F F
19622023-12-10T18:50:34.815Z 17 340-359 12 12 12 18 18 18 F F F
19632023-12-10T18:50:34.815Z 18 360-379 9 9 9 12 12 12 F F F
19642023-12-10T18:50:34.815Z 19 380-399 14 14 14 25 25 25 F F F
19652023-12-10T18:50:34.815Z 20 400-419 14 14 14 25 25 25 F F F
19662023-12-10T18:50:34.815Z 21 420-439 14 14 14 23 23 23 F F F
19672023-12-10T18:50:34.815Z 22 440-459 12 12 12 18 18 18 F F F
19682023-12-10T18:50:34.815Z 23 460-479 15 15 15 26 26 26 F F F
19692023-12-10T18:50:34.815Z 24 480-499 9 9 9 12 12 12 F F F
19702023-12-10T18:50:34.815Z 25 500-519 14 14 14 22 22 22 F F F
19712023-12-10T18:50:34.816Z 26 520-539 14 14 14 22 22 22 F F F
19722023-12-10T18:50:34.816Z 27 540-559 14 14 14 25 25 25 F F F
19732023-12-10T18:50:34.816Z 28 560-579 13 13 13 20 20 20 F F F
19742023-12-10T18:50:34.816Z 29 580-599 12 12 12 19 19 19 F F F
19752023-12-10T18:50:34.816ZMax gen: 15, Max flush: 26
19762023-12-10T18:50:34.816Z
19772023-12-10T18:50:34.816ZBegin loop 8
19782023-12-10T18:50:34.816ZDownstairs to restart: 1
19792023-12-10T18:50:34.816Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.546934773Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223}
19802023-12-10T18:50:34.817Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.547358511Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223}
19812023-12-10T18:50:34.817Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.54738704Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223}
19822023-12-10T18:50:34.817Z{"msg":"Crucible 0f54b0ca-98bc-42cb-97a5-11531cf48663 has session id: 2fff614b-9c3e-4f6a-9294-a4c2b0f59660","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.547445682Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
19832023-12-10T18:50:34.817Z{"msg":"Upstairs opts: Upstairs UUID: 0f54b0ca-98bc-42cb-97a5-11531cf48663, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.547527012Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
19842023-12-10T18:50:34.817ZCrucible runtime is spawned
19852023-12-10T18:50:34.817Z{"msg":"Crucible stats registered with UUID: 0f54b0ca-98bc-42cb-97a5-11531cf48663","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.547607526Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
19862023-12-10T18:50:34.817ZThe guest has requested activation with gen:16
19872023-12-10T18:50:34.817Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.547781457Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660","task":"up_listen"}
19882023-12-10T18:50:34.818Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.547865404Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"looper":"1","session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
19892023-12-10T18:50:34.818Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.547902806Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
19902023-12-10T18:50:34.818Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.547968847Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
19912023-12-10T18:50:34.818Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.548147936Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"looper":"0","session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
19922023-12-10T18:50:34.818Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.548313531Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"looper":"2","session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
19932023-12-10T18:50:34.818Z{"msg":"[1] 0f54b0ca-98bc-42cb-97a5-11531cf48663 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.548402326Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"looper":"1","session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
19942023-12-10T18:50:34.818Z{"msg":"[2] 0f54b0ca-98bc-42cb-97a5-11531cf48663 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.548472525Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"looper":"2","session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
19952023-12-10T18:50:34.818Z{"msg":"[0] 0f54b0ca-98bc-42cb-97a5-11531cf48663 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.548586207Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"looper":"0","session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
19962023-12-10T18:50:34.818Z{"msg":"0f54b0ca-98bc-42cb-97a5-11531cf48663 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.548761603Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
19972023-12-10T18:50:34.818Z{"msg":"Set desired generation to :16","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.548847149Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
19982023-12-10T18:50:34.818Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.549007097Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
19992023-12-10T18:50:34.818Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.549168529Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20002023-12-10T18:50:34.818Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.54955186Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20012023-12-10T18:50:34.818Z{"msg":"[2] 0f54b0ca-98bc-42cb-97a5-11531cf48663 (2fff614b-9c3e-4f6a-9294-a4c2b0f59660) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.549861424Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20022023-12-10T18:50:34.818Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.550024393Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20032023-12-10T18:50:34.818Z{"msg":"[2] client is_active_req TRUE, promote! session 2fff614b-9c3e-4f6a-9294-a4c2b0f59660","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.550089152Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20042023-12-10T18:50:34.818Z{"msg":"[0] 0f54b0ca-98bc-42cb-97a5-11531cf48663 (2fff614b-9c3e-4f6a-9294-a4c2b0f59660) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.550116002Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20052023-12-10T18:50:34.819Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.550295425Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20062023-12-10T18:50:34.819Z{"msg":"[0] client is_active_req TRUE, promote! session 2fff614b-9c3e-4f6a-9294-a4c2b0f59660","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.550358652Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20072023-12-10T18:50:34.819Z{"msg":"[1] 0f54b0ca-98bc-42cb-97a5-11531cf48663 (2fff614b-9c3e-4f6a-9294-a4c2b0f59660) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.550509026Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20082023-12-10T18:50:34.819Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.550598713Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20092023-12-10T18:50:34.819Z{"msg":"[1] client is_active_req TRUE, promote! session 2fff614b-9c3e-4f6a-9294-a4c2b0f59660","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.550644367Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20102023-12-10T18:50:34.819Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.550723931Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20112023-12-10T18:50:34.819Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.550767007Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20122023-12-10T18:50:34.819Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.550812152Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20132023-12-10T18:50:34.819Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.550857983Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20142023-12-10T18:50:34.819Z{"msg":"0f54b0ca-98bc-42cb-97a5-11531cf48663 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.550943144Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20152023-12-10T18:50:34.819Z{"msg":"0f54b0ca-98bc-42cb-97a5-11531cf48663 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.550992917Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20162023-12-10T18:50:34.819Z{"msg":"[0] 0f54b0ca-98bc-42cb-97a5-11531cf48663 (2fff614b-9c3e-4f6a-9294-a4c2b0f59660) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.55153607Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20172023-12-10T18:50:34.819Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.551594768Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20182023-12-10T18:50:34.819Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.551624617Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20192023-12-10T18:50:34.819Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.551713348Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20202023-12-10T18:50:34.819Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:33.55179533Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20212023-12-10T18:50:34.820Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.551856701Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20222023-12-10T18:50:34.820Z{"msg":"[2] 0f54b0ca-98bc-42cb-97a5-11531cf48663 (2fff614b-9c3e-4f6a-9294-a4c2b0f59660) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.551888369Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20232023-12-10T18:50:34.820Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.551943057Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20242023-12-10T18:50:34.820Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:33.551970501Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20252023-12-10T18:50:34.820Z{"msg":"0f54b0ca-98bc-42cb-97a5-11531cf48663 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.552038034Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20262023-12-10T18:50:34.820Z{"msg":"0f54b0ca-98bc-42cb-97a5-11531cf48663 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.552070065Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20272023-12-10T18:50:34.821Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.552127676Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20282023-12-10T18:50:34.821Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.552152211Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20292023-12-10T18:50:34.821Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.552208668Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20302023-12-10T18:50:34.821Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.552236276Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20312023-12-10T18:50:34.821Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.55229651Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20322023-12-10T18:50:34.821Z{"msg":"0f54b0ca-98bc-42cb-97a5-11531cf48663 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.552325449Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20332023-12-10T18:50:34.821Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.552392919Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20342023-12-10T18:50:34.821Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.552431572Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20352023-12-10T18:50:34.821Z{"msg":"[1] 0f54b0ca-98bc-42cb-97a5-11531cf48663 (2fff614b-9c3e-4f6a-9294-a4c2b0f59660) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.552471694Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20362023-12-10T18:50:34.821Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.552555243Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20372023-12-10T18:50:34.822Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:33.552593341Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20382023-12-10T18:50:34.822Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.552630609Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20392023-12-10T18:50:34.822Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.552665997Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20402023-12-10T18:50:34.822Z{"msg":"0f54b0ca-98bc-42cb-97a5-11531cf48663 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.552721742Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20412023-12-10T18:50:34.822Z{"msg":"[0]R flush_numbers[0..12]: [22, 18, 7, 16, 16, 22, 22, 26, 26, 19, 19, 11]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:16
20422023-12-10T18:50:34.822ZWait for a query_work_queue command to finish before sending IO
20432023-12-10T18:50:34.823Z,"time":"2023-12-10T18:50:33.552762196Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20442023-12-10T18:50:34.823Z{"msg":"[0]R generation[0..12]: [14, 12, 5, 10, 10, 14, 14, 15, 15, 12, 12, 8]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.552898613Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20452023-12-10T18:50:34.823Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.552942838Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20462023-12-10T18:50:34.831Z{"msg":"[1]R flush_numbers[0..12]: [22, 18, 7, 16, 16, 22, 22, 26, 26, 19, 19, 11]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.553041199Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20472023-12-10T18:50:34.831Z{"msg":"[1]R generation[0..12]: [14, 12, 5, 10, 10, 14, 14, 15, 15, 12, 12, 8]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.553081242Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20482023-12-10T18:50:34.831Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.553148712Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20492023-12-10T18:50:34.831Z{"msg":"[2]R flush_numbers[0..12]: [22, 18, 7, 16, 16, 22, 22, 26, 26, 19, 19, 11]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.553191042Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20502023-12-10T18:50:34.831Z{"msg":"[2]R generation[0..12]: [14, 12, 5, 10, 10, 14, 14, 15, 15, 12, 12, 8]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.553269375Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20512023-12-10T18:50:34.831Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.553312362Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20522023-12-10T18:50:34.831Z{"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.553351172Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20532023-12-10T18:50:34.831Z{"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.553389822Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20542023-12-10T18:50:34.832Z{"msg":"Next flush: 27","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.553454063Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20552023-12-10T18:50:34.832Z{"msg":"All extents matchRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
20562023-12-10T18:50:34.832Z","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.553537072Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20572023-12-10T18:50:34.832Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.553566123Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20582023-12-10T18:50:34.833Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.553594085Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20592023-12-10T18:50:34.833Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.553617098Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20602023-12-10T18:50:34.833Z{"msg":"0f54b0ca-98bc-42cb-97a5-11531cf48663 is now active with session: 2fff614b-9c3e-4f6a-9294-a4c2b0f59660","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.553660962Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20612023-12-10T18:50:34.833Z{"msg":"0f54b0ca-98bc-42cb-97a5-11531cf48663 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.553684334Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20622023-12-10T18:50:34.833Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.553725134Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20632023-12-10T18:50:34.833Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.553747157Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20642023-12-10T18:50:34.833Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.553786965Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20652023-12-10T18:50:34.834Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.553808833Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20662023-12-10T18:50:34.835Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.553848037Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20672023-12-10T18:50:34.835Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.553869775Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20682023-12-10T18:50:34.835Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.553908696Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20692023-12-10T18:50:34.835Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.55393038Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20702023-12-10T18:50:34.835Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.553969462Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20712023-12-10T18:50:34.835Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:33.55399119Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223Loading write count information from file "/tmp/test_repair_verify.data"
20722023-12-10T18:50:34.835Z,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20732023-12-10T18:50:34.835Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30Run Repair workload
20742023-12-10T18:50:34.835Z,"time":"2023-12-10T18:50:33.55412287Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1223,"session_id":"2fff614b-9c3e-4f6a-9294-a4c2b0f59660"}
20752023-12-10T18:50:34.835Z01/30 Write block 80 len 28672 data: 3 3 3 3 2 3 4
20762023-12-10T18:50:34.835Z02/30 Flush
20772023-12-10T18:50:34.836Z03/30 Write block 191 len 28672 data: 2 2 3 3 4 4 5
20782023-12-10T18:50:34.836Z04/30 Read block 224 len 36864
20792023-12-10T18:50:34.836Z05/30 Write block 126 len 24576 data: 3 2 2 2 2 2
20802023-12-10T18:50:34.836Z06/30 Flush
20812023-12-10T18:50:34.836Z07/30 Write block 221 len 24576 data: 2 2 2 2 2 2
20822023-12-10T18:50:34.836Z08/30 Write block 46 len 20480 data: 3 3 3 3 3
20832023-12-10T18:50:34.836Z09/30 Write block 214 len 36864 data: 2 2 3 2 2 2 2 3 3
20842023-12-10T18:50:34.836Z10/30 Flush
20852023-12-10T18:50:34.836Z11/30 Write block 349 len 20480 data: 4 3 3 4 4
20862023-12-10T18:50:34.836Z12/30 Read block 95 len 36864
20872023-12-10T18:50:34.836Z13/30 Write block 375 len 24576 data: 2 2 2 3 3 4
20882023-12-10T18:50:34.836Z14/30 Write block 383 len 32768 data: 3 3 3 3 4 4 4 5
20892023-12-10T18:50:34.836Z15/30 Flush
20902023-12-10T18:50:34.836Z16/30 Write block 68 len 20480 data: 4 4 3 3 2
20912023-12-10T18:50:34.836Z17/30 Read block 204 len 12288
20922023-12-10T18:50:34.836Z18/30 Read block 585 len 12288
20932023-12-10T18:50:34.836Z19/30 Write block 106 len 20480 data: 3 2 2 2 2
20942023-12-10T18:50:34.837Z20/30 Read block 529 len 40960
20952023-12-10T18:50:34.837Z21/30 Read block 495 len 28672
20962023-12-10T18:50:34.837Z22/30 Write block 188 len 20480 data: 2 2 2 3 3
20972023-12-10T18:50:34.837Z23/30 Write block 268 len 36864 data: 2 2 2 2 2 3 4 4 4
20982023-12-10T18:50:34.837Z24/30 Read block 323 len 40960
20992023-12-10T18:50:34.862Z25/30 Flush
21002023-12-10T18:50:34.862Z26/30 Write block 384 len 20480 data: 4 4 4 5 5
21012023-12-10T18:50:34.862Z27/30 Write block 534 len 8192 data: 4 4
21022023-12-10T18:50:34.862Z28/30 Write block 596 len 8192 data: 3 3
21032023-12-10T18:50:34.862Z29/30 Read block 544 len 16384
21042023-12-10T18:50:34.862Z30/30 Write block 121 len 36864 data: 4 4 4 4 3 4 3 3 3
21052023-12-10T18:50:34.862Z----------------------------------------------------------------
21062023-12-10T18:50:34.862Z Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30
21072023-12-10T18:50:34.862ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
21082023-12-10T18:50:34.862Z 1 Acked 1000 Write 7 Done Sent Done false
21092023-12-10T18:50:34.863Z 2 Acked 1001 Flush 0 Done Sent Done false
21102023-12-10T18:50:34.863Z 3 Acked 1002 Write 7 Done Sent Done false
21112023-12-10T18:50:34.863Z 4 Acked 1003 Read 9 Done Sent Done false
21122023-12-10T18:50:34.863Z 5 Acked 1004 Write 6 Done Sent Done false
21132023-12-10T18:50:34.863Z 6 Acked 1005 Flush 0 Done Sent Done false
21142023-12-10T18:50:34.863Z 7 Acked 1006 Write 6 Done Sent Done false
21152023-12-10T18:50:34.863Z 8 Acked 1007 Write 5 Done Sent Done false
21162023-12-10T18:50:34.863Z 9 Acked 1008 Write 9 Done Sent Done false
21172023-12-10T18:50:34.863Z 10 Acked 1009 Flush 0 Done Sent Done false
21182023-12-10T18:50:34.863Z 11 Acked 1010 Write 5 Done Sent Done false
21192023-12-10T18:50:34.863Z 12 Acked 1011 Read 9 Done Sent Done false
21202023-12-10T18:50:34.863Z 13 Acked 1012 Write 6 Done Sent Done false
21212023-12-10T18:50:34.863Z 14 Acked 1013 Write 8 Done Sent Done false
21222023-12-10T18:50:34.863Z 15 Acked 1014 Flush 0 Done Sent Done false
21232023-12-10T18:50:34.863Z 16 Acked 1015 Write 5 Done Sent Done false
21242023-12-10T18:50:34.863Z 17 Acked 1016 Read 3 Done Sent Done false
21252023-12-10T18:50:34.863Z 18 Acked 1017 Read 3 Done Sent Done false
21262023-12-10T18:50:34.863Z 19 Acked 1018 Write 5 Done Sent Done false
21272023-12-10T18:50:34.864Z 20 Acked 1019 Read 10 Done Sent Done false
21282023-12-10T18:50:34.864Z 21 Acked 1020 Read 7 Done Sent Done false
21292023-12-10T18:50:34.864Z 22 Acked 1021 Write 5 Done Sent Done false
21302023-12-10T18:50:34.864Z 23 Acked 1022 Write 9 Done Sent Done false
21312023-12-10T18:50:34.864Z 24 Acked 1023 Read 10 Done Sent Done false
21322023-12-10T18:50:34.864Z 25 Acked 1024 Flush 0 Done Sent Done false
21332023-12-10T18:50:34.864Z 26 Acked 1025 Write 5 Done Sent Done false
21342023-12-10T18:50:34.864Z 27 Acked 1026 Write 2 Done Sent Done false
21352023-12-10T18:50:34.864Z 28 Acked 1027 Write 2 Done Sent Done false
21362023-12-10T18:50:34.864Z 29 Acked 1028 Read 4 Done Sent Done false
21372023-12-10T18:50:34.864Z 30 Acked 1029 Write 9 Sent Sent Sent false
21382023-12-10T18:50:34.864Z STATES DS:0 DS:1 DS:2 TOTAL
21392023-12-10T18:50:34.864Z New 0 0 0 0
21402023-12-10T18:50:34.864Z Sent 1 30 1 32
21412023-12-10T18:50:34.864Z Done 29 0 29 58
21422023-12-10T18:50:34.864Z Skipped 0 0 0 0
21432023-12-10T18:50:34.864Z Error 0 0 0 0
21442023-12-10T18:50:34.864ZLast Flush: 1024 0 1024
21452023-12-10T18:50:34.865ZDownstairs last five completed:
21462023-12-10T18:50:34.865ZUpstairs last five completed: 30 29 28 27 26 25
21472023-12-10T18:50:34.865ZWrote out file "/tmp/test_repair_verify.data"
21482023-12-10T18:50:34.868Z
21492023-12-10T18:50:34.868ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
21502023-12-10T18:50:34.871ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21512023-12-10T18:50:34.872Z 0 000-019 14 14 14 22 22 22 F F F
21522023-12-10T18:50:34.872Z 1 020-039 12 12 12 18 18 18 F F F
21532023-12-10T18:50:34.872Z 2 040-059 16 5 16 29 7 29 F F F <---
21542023-12-10T18:50:34.872Z 3 060-079 16 10 16 31 16 31 F F F <---
21552023-12-10T18:50:34.872Z 4 080-099 16 10 16 27 16 27 F F F <---
21562023-12-10T18:50:34.872Z 5 100-119 16 14 16 31 22 31 F F F <---
21572023-12-10T18:50:34.872Z 6 120-139 16 14 16 28 22 28 T F T <---
21582023-12-10T18:50:34.872Z 7 140-159 15 15 15 26 26 26 F F F
21592023-12-10T18:50:34.872Z 8 160-179 15 15 15 26 26 26 F F F
21602023-12-10T18:50:34.872Z 9 180-199 16 12 16 31 19 31 F F F <---
21612023-12-10T18:50:34.872Z 10 200-219 16 12 16 29 19 29 F F F <---
21622023-12-10T18:50:34.872Z 11 220-239 16 8 16 29 11 29 F F F <---
21632023-12-10T18:50:34.872Z 12 240-259 12 12 12 19 19 19 F F F
21642023-12-10T18:50:34.872Z 13 260-279 16 8 16 31 11 31 F F F <---
21652023-12-10T18:50:34.872Z 14 280-299 14 14 14 22 22 22 F F F
21662023-12-10T18:50:34.872Z 15 300-319 10 10 10 16 16 16 F F F
21672023-12-10T18:50:34.872Z 16 320-339 14 14 14 22 22 22 F F F
21682023-12-10T18:50:34.873Z 17 340-359 16 12 16 30 18 30 F F F <---
21692023-12-10T18:50:34.873Z 18 360-379 16 9 16 30 12 30 F F F <---
21702023-12-10T18:50:34.873Z 19 380-399 16 14 16 30 25 30 T F T <---
21712023-12-10T18:50:34.873Z 20 400-419 14 14 14 25 25 25 F F F
21722023-12-10T18:50:34.873Z 21 420-439 14 14 14 23 23 23 F F F
21732023-12-10T18:50:34.873Z 22 440-459 12 12 12 18 18 18 F F F
21742023-12-10T18:50:34.873Z 23 460-479 15 15 15 26 26 26 F F F
21752023-12-10T18:50:34.873Z 24 480-499 9 9 9 12 12 12 F F F
21762023-12-10T18:50:34.873Z 25 500-519 14 14 14 22 22 22 F F F
21772023-12-10T18:50:34.873Z 26 520-539 14 14 14 22 22 22 T F T <---
21782023-12-10T18:50:34.873Z 27 540-559 14 14 14 25 25 25 F F F
21792023-12-10T18:50:34.873Z 28 560-579 13 13 13 20 20 20 F F F
21802023-12-10T18:50:34.873Z 29 580-599 12 12 12 19 19 19 T F T <---
21812023-12-10T18:50:34.873ZMax gen: 16, Max flush: 31
21822023-12-10T18:50:34.876ZError: Difference in extent metadata found!
21832023-12-10T18:50:34.877ZA Difference in extent metadata is expected here
21842023-12-10T18:50:34.877ZOn loop 8
21852023-12-10T18:50:34.877Z
21862023-12-10T18:50:34.880ZVerifying data now
21872023-12-10T18:50:40.210ZNo change after verify
21882023-12-10T18:50:40.211ZLoop: 8 Downstairs dump after verify (and repair):
21892023-12-10T18:50:40.233ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
21902023-12-10T18:50:40.233Z 0 000-019 14 14 14 22 22 22 F F F
21912023-12-10T18:50:40.233Z 1 020-039 12 12 12 18 18 18 F F F
21922023-12-10T18:50:40.233Z 2 040-059 16 16 16 29 29 29 F F F
21932023-12-10T18:50:40.233Z 3 060-079 16 16 16 31 31 31 F F F
21942023-12-10T18:50:40.233Z 4 080-099 16 16 16 27 27 27 F F F
21952023-12-10T18:50:40.237Z 5 100-119 16 16 16 31 31 31 F F F
21962023-12-10T18:50:40.238Z 6 120-139 17 17 17 32 32 32 F F F
21972023-12-10T18:50:40.238Z 7 140-159 15 15 15 26 26 26 F F F
21982023-12-10T18:50:40.238Z 8 160-179 15 15 15 26 26 26 F F F
21992023-12-10T18:50:40.238Z 9 180-199 16 16 16 31 31 31 F F F
22002023-12-10T18:50:40.238Z 10 200-219 16 16 16 29 29 29 F F F
22012023-12-10T18:50:40.239Z 11 220-239 16 16 16 29 29 29 F F F
22022023-12-10T18:50:40.239Z 12 240-259 12 12 12 19 19 19 F F F
22032023-12-10T18:50:40.239Z 13 260-279 16 16 16 31 31 31 F F F
22042023-12-10T18:50:40.239Z 14 280-299 14 14 14 22 22 22 F F F
22052023-12-10T18:50:40.239Z 15 300-319 10 10 10 16 16 16 F F F
22062023-12-10T18:50:40.239Z 16 320-339 14 14 14 22 22 22 F F F
22072023-12-10T18:50:40.239Z 17 340-359 16 16 16 30 30 30 F F F
22082023-12-10T18:50:40.239Z 18 360-379 16 16 16 30 30 30 F F F
22092023-12-10T18:50:40.239Z 19 380-399 17 17 17 32 32 32 F F F
22102023-12-10T18:50:40.239Z 20 400-419 14 14 14 25 25 25 F F F
22112023-12-10T18:50:40.239Z 21 420-439 14 14 14 23 23 23 F F F
22122023-12-10T18:50:40.239Z 22 440-459 12 12 12 18 18 18 F F F
22132023-12-10T18:50:40.239Z 23 460-479 15 15 15 26 26 26 F F F
22142023-12-10T18:50:40.240Z 24 480-499 9 9 9 12 12 12 F F F
22152023-12-10T18:50:40.240Z 25 500-519 14 14 14 22 22 22 F F F
22162023-12-10T18:50:40.240Z 26 520-539 17 17 17 32 32 32 F F F
22172023-12-10T18:50:40.240Z 27 540-559 14 14 14 25 25 25 F F F
22182023-12-10T18:50:40.240Z 28 560-579 13 13 13 20 20 20 F F F
22192023-12-10T18:50:40.240Z 29 580-599 17 17 17 32 32 32 F F F
22202023-12-10T18:50:40.240ZMax gen: 17, Max flush: 32
22212023-12-10T18:50:40.244Z
22222023-12-10T18:50:40.244ZBegin loop 9
22232023-12-10T18:50:40.244ZDownstairs to restart: 2
22242023-12-10T18:50:40.272Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.046438002Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231}
22252023-12-10T18:50:40.272Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.046909542Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231}
22262023-12-10T18:50:40.278Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.046978852Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231}
22272023-12-10T18:50:40.279Z{"msg":"Crucible 1467aae1-e19a-4994-925a-ec7c41125bbe has session id: 0af0a844-ed7b-4724-a754-a4328147f938","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.047052942Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22282023-12-10T18:50:40.279Z{"msg":"Crucible runtime is spawned
22292023-12-10T18:50:40.279ZUpstairs opts: Upstairs UUID: 1467aae1-e19a-4994-925a-ec7c41125bbe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.047294399Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22302023-12-10T18:50:40.279Z{"msg":"Crucible stats registered with UUID: 1467aae1-e19a-4994-925a-ec7c41125bbe","v":0,"name":"crucible","level":30,"time"The guest has requested activation with gen:18
22312023-12-10T18:50:40.279Z:"2023-12-10T18:50:39.047334694Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22322023-12-10T18:50:40.279Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.047514176Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"looper":"0","session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22332023-12-10T18:50:40.279Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.047558043Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938","task":"up_listen"}
22342023-12-10T18:50:40.280Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.04762568Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22352023-12-10T18:50:40.280Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.047668917Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22362023-12-10T18:50:40.280Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.047739227Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"looper":"2","session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22372023-12-10T18:50:40.281Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.04776401Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"looper":"1","session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22382023-12-10T18:50:40.281Z{"msg":"[0] 1467aae1-e19a-4994-925a-ec7c41125bbe looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.047786259Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"looper":"0","session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22392023-12-10T18:50:40.281Z{"msg":"[1] 1467aae1-e19a-4994-925a-ec7c41125bbe looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.04791234Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"looper":"1","session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22402023-12-10T18:50:40.281Z{"msg":"1467aae1-e19a-4994-925a-ec7c41125bbe active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.048305603Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22412023-12-10T18:50:40.281Z{"msg":"Set desired generation to :18","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.048360087Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22422023-12-10T18:50:40.281Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.048384784Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22432023-12-10T18:50:40.281Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.048742705Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22442023-12-10T18:50:40.281Z{"msg":"[0] 1467aae1-e19a-4994-925a-ec7c41125bbe (0af0a844-ed7b-4724-a754-a4328147f938) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.049119761Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22452023-12-10T18:50:40.281Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.049199937Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22462023-12-10T18:50:40.281Z{"msg":"[0] client is_active_req TRUE, promote! session 0af0a844-ed7b-4724-a754-a4328147f938","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.049226521Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22472023-12-10T18:50:40.281Z{"msg":"[1] 1467aae1-e19a-4994-925a-ec7c41125bbe (0af0a844-ed7b-4724-a754-a4328147f938) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.049436112Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22482023-12-10T18:50:40.281Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.049567711Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22492023-12-10T18:50:40.282Z{"msg":"[1] client is_active_req TRUE, promote! session 0af0a844-ed7b-4724-a754-a4328147f938","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.049635027Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22502023-12-10T18:50:40.282Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.050053492Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22512023-12-10T18:50:40.282Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.050211306Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22522023-12-10T18:50:40.282Z{"msg":"1467aae1-e19a-4994-925a-ec7c41125bbe WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.050239948Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22532023-12-10T18:50:40.282Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.050261952Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22542023-12-10T18:50:40.282Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.050336019Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22552023-12-10T18:50:40.282Z{"msg":"1467aae1-e19a-4994-925a-ec7c41125bbe WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.050384407Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22562023-12-10T18:50:40.282Z{"msg":"[0] 1467aae1-e19a-4994-925a-ec7c41125bbe (0af0a844-ed7b-4724-a754-a4328147f938) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.050942667Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22572023-12-10T18:50:40.282Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.051018086Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22582023-12-10T18:50:40.282Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:39.051043733Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22592023-12-10T18:50:40.282Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.051064239Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22602023-12-10T18:50:40.282Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.051085277Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22612023-12-10T18:50:40.283Z{"msg":"1467aae1-e19a-4994-925a-ec7c41125bbe WaitQuorum WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.051106458Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22622023-12-10T18:50:40.283Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.051137189Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22632023-12-10T18:50:40.287Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.051158337Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22642023-12-10T18:50:40.287Z{"msg":"[1] 1467aae1-e19a-4994-925a-ec7c41125bbe (0af0a844-ed7b-4724-a754-a4328147f938) WaitQuorum WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.05135073Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22652023-12-10T18:50:40.287Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.05140469Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22662023-12-10T18:50:40.287Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:39.0514283Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22672023-12-10T18:50:40.287Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.051469056Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22682023-12-10T18:50:40.287Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.051491893Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22692023-12-10T18:50:40.287Z{"msg":"1467aae1-e19a-4994-925a-ec7c41125bbe WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.051532633Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22702023-12-10T18:50:40.287Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.051555088Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22712023-12-10T18:50:40.288Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:39.051575109Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22722023-12-10T18:50:41.279Z{"msg":"[2] 1467aae1-e19a-4994-925a-ec7c41125bbe looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.048313329Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"looper":"2","session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22732023-12-10T18:50:41.280Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.048398659Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22742023-12-10T18:50:41.295Z{"msg":"[2] 1467aae1-e19a-4994-925a-ec7c41125bbe (0af0a844-ed7b-4724-a754-a4328147f938) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.049552431Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22752023-12-10T18:50:41.295Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.049612535Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22762023-12-10T18:50:41.295Z{"msg":"[2] client is_active_req TRUE, promote! session 0af0a844-ed7b-4724-a754-a4328147f938","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.049638115Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22772023-12-10T18:50:41.295Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.050335402Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22782023-12-10T18:50:41.295Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.050407589Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22792023-12-10T18:50:41.296Z{"msg":"1467aae1-e19a-4994-925a-ec7c41125bbe WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.05044866Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22802023-12-10T18:50:41.296Z{"msg":"[2] 1467aae1-e19a-4994-925a-ec7c41125bbe (0af0a844-ed7b-4724-a754-a4328147f938) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.051131815Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22812023-12-10T18:50:41.296Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.051163342Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22822023-12-10T18:50:41.296Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:40.051185149Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22832023-12-10T18:50:41.296Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.051212569Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22842023-12-10T18:50:41.296Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.051289364Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22852023-12-10T18:50:41.296Z{"msg":"1467aae1-e19a-4994-925a-ec7c41125bbe WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.051327856Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22862023-12-10T18:50:41.296Z{"msg":"[0]R flush_numbers[0..12]: [22, 18, 29, 31, 27, 31, 32, 26, 26, 31, 29, 29]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.051383194Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22872023-12-10T18:50:41.296Z{"msg":"[0]R generation[0..12]: [14, 12, 16, 16, 16, 16, 17, 15, 15, 16, 16, 16]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.05141285Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22882023-12-10T18:50:41.296Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:18
22892023-12-10T18:50:41.296Z,"time":"2023-12-10T18:50:40.051469222Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal",Wait for a query_work_queue command to finish before sending IO
22902023-12-10T18:50:41.296Z"pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22912023-12-10T18:50:41.296Z{"msg":"[1]R flush_numbers[0..12]: [22, 18, 29, 31, 27, 31, 32, 26, 26, 31, 29, 29]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.051501342Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22922023-12-10T18:50:41.296Z{"msg":"[1]R generation[0..12]: [14, 12, 16, 16, 16, 16, 17, 15, 15, 16, 16, 16]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.051528598Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22932023-12-10T18:50:41.296Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.051601983Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22942023-12-10T18:50:41.296Z{"msg":"[2]R flush_numbers[0..12]: [22, 18, 29, 31, 27, 31, 32, 26, 26, 31, 29, 29]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.051643217Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22952023-12-10T18:50:41.296Z{"msg":"[2]R generation[0..12]: [14, 12, 16, 16, 16, 16, 17, 15, 15, 16, 16, 16]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.051680044Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22962023-12-10T18:50:41.297Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.051745882Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22972023-12-10T18:50:41.297Z{"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.051769904Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22982023-12-10T18:50:41.297Z{"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.051798231Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
22992023-12-10T18:50:41.297Z{"msg":"Next flush: 33","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.051852757Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
23002023-12-10T18:50:41.297Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.051876717Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
23012023-12-10T18:50:41.297Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.051934762Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
23022023-12-10T18:50:41.297Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.051960417Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
23032023-12-10T18:50:41.297Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.052019607Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
23042023-12-10T18:50:41.297Z{"msg":"1467aae1-e19a-4994-925a-ec7c41125bbe is now active with session: 0af0a844-ed7b-4724-a754-a4328147f938","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.052046075Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
23052023-12-10T18:50:41.297Z{"msg":"1467aae1-e19a-4994-925a-ec7c41125bbe Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.052068996Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
23062023-12-10T18:50:41.297Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.052089687Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
23072023-12-10T18:50:41.297Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.052150113Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
23082023-12-10T18:50:41.297Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.052173928Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
23092023-12-10T18:50:41.297Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.052193766Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
23102023-12-10T18:50:41.297Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.052226706Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
23112023-12-10T18:50:41.297Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.052285236Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
23122023-12-10T18:50:41.297Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.052307357Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
23132023-12-10T18:50:41.297Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.052327414Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
23142023-12-10T18:50:41.298Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.052386785Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
23152023-12-10T18:50:41.298Zip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
23162023-12-10T18:50:41.298Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.052464659Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
23172023-12-10T18:50:41.298Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:40.052490732Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1231,"session_id":"0af0a844-ed7b-4724-a754-a4328147f938"}
23182023-12-10T18:50:41.298ZLoading write count information from file "/tmp/test_repair_verify.data"
23192023-12-10T18:50:41.298ZRun Repair workload
23202023-12-10T18:50:41.298Z01/30 Flush
23212023-12-10T18:50:41.298Z02/30 Write block 441 len 4096 data: 5
23222023-12-10T18:50:41.298Z03/30 Read block 518 len 16384
23232023-12-10T18:50:41.298Z04/30 Read block 83 len 36864
23242023-12-10T18:50:41.298Z05/30 Write block 205 len 12288 data: 2 2 2
23252023-12-10T18:50:41.302Z06/30 Write block 239 len 20480 data: 5 5 4 4 4
23262023-12-10T18:50:41.303Z07/30 Read block 183 len 36864
23272023-12-10T18:50:41.303Z08/30 Read block 389 len 8192
23282023-12-10T18:50:41.303Z09/30 Read block 454 len 36864
23292023-12-10T18:50:41.303Z10/30 Write block 93 len 12288 data: 3 3 3
23302023-12-10T18:50:41.303Z11/30 Flush
23312023-12-10T18:50:41.303Z12/30 Write block 68 len 36864 data: 5 5 4 4 3 2 3 3 2
23322023-12-10T18:50:41.303Z13/30 Write block 242 len 24576 data: 5 5 5 5 4 4
23332023-12-10T18:50:41.307Z14/30 Flush
23342023-12-10T18:50:41.307Z15/30 Flush
23352023-12-10T18:50:41.307Z16/30 Write block 212 len 28672 data: 2 2 3 3 4 3 3
23362023-12-10T18:50:41.307Z17/30 Read block 131 len 36864
23372023-12-10T18:50:41.310Z18/30 Write block 177 len 4096 data: 4
23382023-12-10T18:50:41.313Z19/30 Write block 36 len 8192 data: 2 3
23392023-12-10T18:50:41.313Z20/30 Read block 51 len 12288
23402023-12-10T18:50:41.314Z21/30 Read block 100 len 32768
23412023-12-10T18:50:41.317Z22/30 Read block 534 len 36864
23422023-12-10T18:50:41.320Z23/30 Write block 293 len 12288 data: 2 2 2
23432023-12-10T18:50:41.320Z24/30 Read block 284 len 32768
23442023-12-10T18:50:41.323Z25/30 Read block 419 len 40960
23452023-12-10T18:50:41.326Z26/30 Flush
23462023-12-10T18:50:41.330Z27/30 Write block 115 len 40960 data: 3 4 4 5 5 5 5 5 5 5
23472023-12-10T18:50:41.333Z28/30 Read block 198 len 8192
23482023-12-10T18:50:41.333Z29/30 Write block 262 len 16384 data: 3 3 3 3
23492023-12-10T18:50:41.337Z30/30 Write block 496 len 8192 data: 2 2
23502023-12-10T18:50:41.337Z----------------------------------------------------------------
23512023-12-10T18:50:41.337Z Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:29
23522023-12-10T18:50:41.337ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
23532023-12-10T18:50:41.337Z 2 Acked 1001 Write 1 Done Done Done false
23542023-12-10T18:50:41.337Z 3 Acked 1002 Read 4 Done Done Done false
23552023-12-10T18:50:41.337Z 4 Acked 1003 Read 9 Done Done Done false
23562023-12-10T18:50:41.337Z 5 Acked 1004 Write 3 Done Done Done false
23572023-12-10T18:50:41.337Z 6 Acked 1005 Write 5 Done Done Done false
23582023-12-10T18:50:41.337Z 7 Acked 1006 Read 9 Done Done Sent false
23592023-12-10T18:50:41.337Z 8 Acked 1007 Read 2 Done Done Sent false
23602023-12-10T18:50:41.337Z 9 Acked 1008 Read 9 Done Done Sent false
23612023-12-10T18:50:41.337Z 10 Acked 1009 Write 3 Done Done Sent false
23622023-12-10T18:50:41.337Z 11 Acked 1010 Flush 0 Done Done Sent false
23632023-12-10T18:50:41.337Z 12 Acked 1011 Write 9 Done Done Sent false
23642023-12-10T18:50:41.337Z 13 Acked 1012 Write 6 Done Done Sent false
23652023-12-10T18:50:41.338Z 14 Acked 1013 Flush 0 Done Done Sent false
23662023-12-10T18:50:41.338Z 15 Acked 1014 Flush 0 Done Done Sent false
23672023-12-10T18:50:41.338Z 16 Acked 1015 Write 7 Done Done Sent false
23682023-12-10T18:50:41.338Z 17 Acked 1016 Read 9 Done Done Sent false
23692023-12-10T18:50:41.338Z 18 Acked 1017 Write 1 Done Done Sent false
23702023-12-10T18:50:41.338Z 19 Acked 1018 Write 2 Done Done Sent false
23712023-12-10T18:50:41.338Z 20 Acked 1019 Read 3 Done Done Sent false
23722023-12-10T18:50:41.338Z 21 Acked 1020 Read 8 Done Done Sent false
23732023-12-10T18:50:41.338Z 22 Acked 1021 Read 9 Done Done Sent false
23742023-12-10T18:50:41.338Z 23 Acked 1022 Write 3 Done Done Sent false
23752023-12-10T18:50:41.338Z 24 Acked 1023 Read 8 Done Done Sent false
23762023-12-10T18:50:41.338Z 25 Acked 1024 Read 10 Done Done Sent false
23772023-12-10T18:50:41.338Z 26 Acked 1025 Flush 0 Done Done Sent false
23782023-12-10T18:50:41.338Z 27 Acked 1026 Write 10 Done Done Sent false
23792023-12-10T18:50:41.338Z 28 Acked 1027 Read 2 Done Done Sent false
23802023-12-10T18:50:41.338Z 29 Acked 1028 Write 4 Done Done Sent false
23812023-12-10T18:50:41.338Z 30 Acked 1029 Write 2 Sent Sent Sent false
23822023-12-10T18:50:41.339Z STATES DS:0 DS:1 DS:2 TOTAL
23832023-12-10T18:50:41.339Z New 0 0 0 0
23842023-12-10T18:50:41.339Z Sent 1 1 24 26
23852023-12-10T18:50:41.339Z Done 28 28 5 61
23862023-12-10T18:50:41.339Z Skipped 0 0 0 0
23872023-12-10T18:50:41.339Z Error 0 0 0 0
23882023-12-10T18:50:41.339ZLast Flush: 1025 1025 1000
23892023-12-10T18:50:41.339ZDownstairs last five completed: 1000
23902023-12-10T18:50:41.339ZUpstairs last five completed: 30 29 28 27 26 25
23912023-12-10T18:50:41.339ZWrote out file "/tmp/test_repair_verify.data"
23922023-12-10T18:50:41.343Z
23932023-12-10T18:50:41.349ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
23942023-12-10T18:50:41.376ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23952023-12-10T18:50:41.376Z 0 000-019 14 14 14 22 22 22 F F F
23962023-12-10T18:50:41.376Z 1 020-039 18 18 12 37 37 18 F F F <---
23972023-12-10T18:50:41.376Z 2 040-059 16 16 16 29 29 29 F F F
23982023-12-10T18:50:41.376Z 3 060-079 18 18 16 35 35 31 F F F <---
23992023-12-10T18:50:41.380Z 4 080-099 18 18 16 34 34 27 F F F <---
24002023-12-10T18:50:41.380Z 5 100-119 16 16 16 31 31 31 T T F <---
24012023-12-10T18:50:41.380Z 6 120-139 17 17 17 32 32 32 T T F <---
24022023-12-10T18:50:41.380Z 7 140-159 15 15 15 26 26 26 F F F
24032023-12-10T18:50:41.380Z 8 160-179 18 18 15 37 37 26 F F F <---
24042023-12-10T18:50:41.380Z 9 180-199 16 16 16 31 31 31 F F F
24052023-12-10T18:50:41.380Z 10 200-219 18 18 16 37 37 29 F F T <---
24062023-12-10T18:50:41.380Z 11 220-239 18 18 16 34 34 29 F F T <---
24072023-12-10T18:50:41.380Z 12 240-259 18 18 12 35 35 19 F F T <---
24082023-12-10T18:50:41.380Z 13 260-279 16 16 16 31 31 31 T T F <---
24092023-12-10T18:50:41.381Z 14 280-299 18 18 14 37 37 22 F F F <---
24102023-12-10T18:50:41.381Z 15 300-319 10 10 10 16 16 16 F F F
24112023-12-10T18:50:41.381Z 16 320-339 14 14 14 22 22 22 F F F
24122023-12-10T18:50:41.381Z 17 340-359 16 16 16 30 30 30 F F F
24132023-12-10T18:50:41.382Z 18 360-379 16 16 16 30 30 30 F F F
24142023-12-10T18:50:41.382Z 19 380-399 17 17 17 32 32 32 F F F
24152023-12-10T18:50:41.382Z 20 400-419 14 14 14 25 25 25 F F F
24162023-12-10T18:50:41.382Z 21 420-439 14 14 14 23 23 23 F F F
24172023-12-10T18:50:41.382Z 22 440-459 18 18 12 34 34 18 F F T <---
24182023-12-10T18:50:41.382Z 23 460-479 15 15 15 26 26 26 F F F
24192023-12-10T18:50:41.382Z 24 480-499 9 9 9 12 12 12 T T F <---
24202023-12-10T18:50:41.382Z 25 500-519 14 14 14 22 22 22 F F F
24212023-12-10T18:50:41.382Z 26 520-539 17 17 17 32 32 32 F F F
24222023-12-10T18:50:41.382Z 27 540-559 14 14 14 25 25 25 F F F
24232023-12-10T18:50:41.382Z 28 560-579 13 13 13 20 20 20 F F F
24242023-12-10T18:50:41.382Z 29 580-599 17 17 17 32 32 32 F F F
24252023-12-10T18:50:41.384ZMax gen: 18, Max flush: 37
24262023-12-10T18:50:41.384ZError: Difference in extent metadata found!
24272023-12-10T18:50:41.388ZA Difference in extent metadata is expected here
24282023-12-10T18:50:41.388ZOn loop 9
24292023-12-10T18:50:41.388Z
24302023-12-10T18:50:41.388ZVerifying data now
24312023-12-10T18:50:47.665ZNo change after verify
24322023-12-10T18:50:47.665ZLoop: 9 Downstairs dump after verify (and repair):
24332023-12-10T18:50:47.688ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
24342023-12-10T18:50:47.689Z 0 000-019 14 14 14 22 22 22 F F F
24352023-12-10T18:50:47.689Z 1 020-039 18 18 18 37 37 37 F F F
24362023-12-10T18:50:47.689Z 2 040-059 16 16 16 29 29 29 F F F
24372023-12-10T18:50:47.689Z 3 060-079 18 18 18 35 35 35 F F F
24382023-12-10T18:50:47.689Z 4 080-099 18 18 18 34 34 34 F F F
24392023-12-10T18:50:47.689Z 5 100-119 19 19 19 38 38 38 F F F
24402023-12-10T18:50:47.694Z 6 120-139 19 19 19 38 38 38 F F F
24412023-12-10T18:50:47.694Z 7 140-159 15 15 15 26 26 26 F F F
24422023-12-10T18:50:47.694Z 8 160-179 18 18 18 37 37 37 F F F
24432023-12-10T18:50:47.694Z 9 180-199 16 16 16 31 31 31 F F F
24442023-12-10T18:50:47.694Z 10 200-219 18 18 18 37 37 37 F F F
24452023-12-10T18:50:47.694Z 11 220-239 18 18 18 34 34 34 F F F
24462023-12-10T18:50:47.694Z 12 240-259 18 18 18 35 35 35 F F F
24472023-12-10T18:50:47.694Z 13 260-279 19 19 19 38 38 38 F F F
24482023-12-10T18:50:47.695Z 14 280-299 18 18 18 37 37 37 F F F
24492023-12-10T18:50:47.695Z 15 300-319 10 10 10 16 16 16 F F F
24502023-12-10T18:50:47.695Z 16 320-339 14 14 14 22 22 22 F F F
24512023-12-10T18:50:47.695Z 17 340-359 16 16 16 30 30 30 F F F
24522023-12-10T18:50:47.695Z 18 360-379 16 16 16 30 30 30 F F F
24532023-12-10T18:50:47.695Z 19 380-399 17 17 17 32 32 32 F F F
24542023-12-10T18:50:47.695Z 20 400-419 14 14 14 25 25 25 F F F
24552023-12-10T18:50:47.695Z 21 420-439 14 14 14 23 23 23 F F F
24562023-12-10T18:50:47.695Z 22 440-459 18 18 18 34 34 34 F F F
24572023-12-10T18:50:47.696Z 23 460-479 15 15 15 26 26 26 F F F
24582023-12-10T18:50:47.696Z 24 480-499 19 19 19 38 38 38 F F F
24592023-12-10T18:50:47.696Z 25 500-519 14 14 14 22 22 22 F F F
24602023-12-10T18:50:47.696Z 26 520-539 17 17 17 32 32 32 F F F
24612023-12-10T18:50:47.696Z 27 540-559 14 14 14 25 25 25 F F F
24622023-12-10T18:50:47.696Z 28 560-579 13 13 13 20 20 20 F F F
24632023-12-10T18:50:47.696Z 29 580-599 17 17 17 32 32 32 F F F
24642023-12-10T18:50:47.696ZMax gen: 19, Max flush: 38
24652023-12-10T18:50:47.697Z
24662023-12-10T18:50:47.697ZBegin loop 10
24672023-12-10T18:50:47.697ZDownstairs to restart: 1
24682023-12-10T18:50:47.723Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.497683566Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239}
24692023-12-10T18:50:47.724Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.498109092Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239}
24702023-12-10T18:50:47.725Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.498137269Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239}
24712023-12-10T18:50:47.731Z{"msg":"Crucible 83f487b9-5e0f-43ac-bc65-cd31eb81bf99 has session id: 9cf5e07f-8a98-473f-9317-53535fe52d8f","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.498170159Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
24722023-12-10T18:50:47.731Z{"msg":"Upstairs opts: Upstairs UUID: 83f487b9-5e0f-43ac-bc65-cd31eb81bf99, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.498287366Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
24732023-12-10T18:50:47.732Z{"msg":"Crucible stats registered with UUID: 83f487b9-5e0f-43ac-bc65-cd31eb81bf99","v":0,"name":"crucible","level":30Crucible runtime is spawned
24742023-12-10T18:50:47.732Z,"time":"2023-12-10T18:50:46.498331627Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
24752023-12-10T18:50:47.732ZThe guest has requested activation with gen:20
24762023-12-10T18:50:47.732Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.498550422Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"looper":"1","session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
24772023-12-10T18:50:47.732Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.49865882Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"looper":"0","session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
24782023-12-10T18:50:47.732Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.498700475Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"looper":"2","session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
24792023-12-10T18:50:47.732Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.4987382Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f","task":"up_listen"}
24802023-12-10T18:50:47.732Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.498827962Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
24812023-12-10T18:50:47.732Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.498868518Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
24822023-12-10T18:50:47.733Z{"msg":"[2] 83f487b9-5e0f-43ac-bc65-cd31eb81bf99 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.499169118Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"looper":"2","session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
24832023-12-10T18:50:47.733Z{"msg":"[0] 83f487b9-5e0f-43ac-bc65-cd31eb81bf99 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.499258944Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"looper":"0","session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
24842023-12-10T18:50:47.733Z{"msg":"83f487b9-5e0f-43ac-bc65-cd31eb81bf99 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.499304171Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
24852023-12-10T18:50:47.733Z{"msg":"Set desired generation to :20","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.499371791Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
24862023-12-10T18:50:47.733Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.499441911Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
24872023-12-10T18:50:47.733Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.499598041Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
24882023-12-10T18:50:47.733Z{"msg":"[2] 83f487b9-5e0f-43ac-bc65-cd31eb81bf99 (9cf5e07f-8a98-473f-9317-53535fe52d8f) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.499944473Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
24892023-12-10T18:50:47.733Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.500087853Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
24902023-12-10T18:50:47.733Z{"msg":"[2] client is_active_req TRUE, promote! session 9cf5e07f-8a98-473f-9317-53535fe52d8f","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.500131086Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
24912023-12-10T18:50:47.735Z{"msg":"[0] 83f487b9-5e0f-43ac-bc65-cd31eb81bf99 (9cf5e07f-8a98-473f-9317-53535fe52d8f) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.500198936Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
24922023-12-10T18:50:47.735Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.50023662Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
24932023-12-10T18:50:47.735Z{"msg":"[0] client is_active_req TRUE, promote! session 9cf5e07f-8a98-473f-9317-53535fe52d8f","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.500301917Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
24942023-12-10T18:50:47.735Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.500611892Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
24952023-12-10T18:50:47.735Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.500703775Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
24962023-12-10T18:50:47.735Z{"msg":"83f487b9-5e0f-43ac-bc65-cd31eb81bf99 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.500880249Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
24972023-12-10T18:50:47.735Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.500922321Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
24982023-12-10T18:50:47.736Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.500994412Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
24992023-12-10T18:50:47.736Z{"msg":"83f487b9-5e0f-43ac-bc65-cd31eb81bf99 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.501064878Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25002023-12-10T18:50:47.736Z{"msg":"[2] 83f487b9-5e0f-43ac-bc65-cd31eb81bf99 (9cf5e07f-8a98-473f-9317-53535fe52d8f) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.501428134Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25012023-12-10T18:50:47.737Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.501480084Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25022023-12-10T18:50:47.737Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:46.501522313Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25032023-12-10T18:50:47.737Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.50169709Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25042023-12-10T18:50:47.737Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.501834506Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25052023-12-10T18:50:47.737Z{"msg":"83f487b9-5e0f-43ac-bc65-cd31eb81bf99 WaitActive New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.501936818Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25062023-12-10T18:50:47.737Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.501965367Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25072023-12-10T18:50:47.737Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.502030985Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25082023-12-10T18:50:47.737Z{"msg":"[0] 83f487b9-5e0f-43ac-bc65-cd31eb81bf99 (9cf5e07f-8a98-473f-9317-53535fe52d8f) WaitActive New WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.502076238Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25092023-12-10T18:50:47.738Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.502120765Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25102023-12-10T18:50:47.738Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:46.502143289Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25112023-12-10T18:50:47.739Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.502164529Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25122023-12-10T18:50:47.739Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.502185223Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25132023-12-10T18:50:47.739Z{"msg":"83f487b9-5e0f-43ac-bc65-cd31eb81bf99 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.502225835Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25142023-12-10T18:50:47.739Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.502248067Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25152023-12-10T18:50:47.739Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:46.502287372Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25162023-12-10T18:50:48.726Z{"msg":"[1] 83f487b9-5e0f-43ac-bc65-cd31eb81bf99 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.500215856Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"looper":"1","session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25172023-12-10T18:50:48.726Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.500305923Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25182023-12-10T18:50:48.743Z{"msg":"[1] 83f487b9-5e0f-43ac-bc65-cd31eb81bf99 (9cf5e07f-8a98-473f-9317-53535fe52d8f) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.501195905Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25192023-12-10T18:50:48.743Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.501308947Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25202023-12-10T18:50:48.743Z{"msg":"[1] client is_active_req TRUE, promote! session 9cf5e07f-8a98-473f-9317-53535fe52d8f","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.501336226Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25212023-12-10T18:50:48.743Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.501890901Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25222023-12-10T18:50:48.744Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.501947205Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25232023-12-10T18:50:48.744Z{"msg":"83f487b9-5e0f-43ac-bc65-cd31eb81bf99 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.501971957Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25242023-12-10T18:50:48.744Z{"msg":"[1] 83f487b9-5e0f-43ac-bc65-cd31eb81bf99 (9cf5e07f-8a98-473f-9317-53535fe52d8f) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.502607723Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25252023-12-10T18:50:48.744Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.502657525Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25262023-12-10T18:50:48.744Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:47.502680555Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25272023-12-10T18:50:48.744Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.502735487Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25282023-12-10T18:50:48.745Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.502758487Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25292023-12-10T18:50:48.745Z{"msg":"83f487b9-5e0f-43ac-bc65-cd31eb81bf99 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.502780043Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25302023-12-10T18:50:48.745Z{"msg":"[0]R flush_numbers[0..12]: [22, 37, 29, 35, 34, 38, 38, 26, 37, 31, 37, 34]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.502821489Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25312023-12-10T18:50:48.745Z{"msg":"[0]R generation[0..12]: [14, 18, 16, 18, 18, 19, 19, 15, 18, 16, 18, 18]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.502845166Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25322023-12-10T18:50:48.745Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.502887282Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25332023-12-10T18:50:48.745Z{"msg":"[1]R flush_numbers[0..12]: [22, 37, 29, 35, 34, 38, 38, 26, 37, 31, 37, 34]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.502910626Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25342023-12-10T18:50:48.745Z{"msg":"[1]R generation[0..12]: [14, 18, 16, 18, 18, 19, 19, 15, 18, 16, 18, 18]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.502955486Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25352023-12-10T18:50:48.745Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.50297958Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25362023-12-10T18:50:48.746Z{"msg":"[2]R flush_numbers[0..12]: [22, 37, 29, 35, 34, 38, 38, 26, 37, 31, 37, 34]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:20
25372023-12-10T18:50:48.746ZWait for a query_work_queue command to finish before sending IO
25382023-12-10T18:50:48.746Z,"time":"2023-12-10T18:50:47.503002435Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25392023-12-10T18:50:48.746Z{"msg":"[2]R generation[0..12]: [14, 18, 16, 18, 18, 19, 19, 15, 18, 16, 18, 18]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.50324843Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25402023-12-10T18:50:48.746Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.503299863Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25412023-12-10T18:50:48.747Z{"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.503363121Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25422023-12-10T18:50:48.747Z{"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.503402143Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25432023-12-10T18:50:48.747Z{"msg":"Next flush: 39","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.503437182Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25442023-12-10T18:50:48.747Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.503490505Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25452023-12-10T18:50:48.747Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.503529154Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25462023-12-10T18:50:48.748Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.503586993Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25472023-12-10T18:50:48.748Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.503626029Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25482023-12-10T18:50:48.748Z{"msg":"83f487b9-5e0f-43ac-bc65-cd31eb81bf99 is now active with session: 9cf5e07f-8a98-473f-9317-53535fe52d8f","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.503683991Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25492023-12-10T18:50:48.748Z{"msg":"83f487b9-5e0f-43ac-bc65-cd31eb81bf99 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.503723463Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25502023-12-10T18:50:48.748Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.503789154Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25512023-12-10T18:50:48.748Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.503827091Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25522023-12-10T18:50:48.751Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.503881353Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25532023-12-10T18:50:48.751Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
25542023-12-10T18:50:48.751Z[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.503930041Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25552023-12-10T18:50:48.751Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.503967251Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25562023-12-10T18:50:48.751Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.504043743Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25572023-12-10T18:50:48.751Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.504084129Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25582023-12-10T18:50:48.752Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.504120193Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25592023-12-10T18:50:48.752Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.504172645Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25602023-12-10T18:50:48.752Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.504210612Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25612023-12-10T18:50:48.752Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:47.504264425Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1239,"session_id":"9cf5e07f-8a98-473f-9317-53535fe52d8f"}
25622023-12-10T18:50:48.753ZLoading write count information from file "/tmp/test_repair_verify.data"
25632023-12-10T18:50:48.753ZRun Repair workload
25642023-12-10T18:50:48.753Z01/30 Write block 31 len 24576 data: 2 2 2 2 2 3
25652023-12-10T18:50:48.761Z02/30 Read block 312 len 20480
25662023-12-10T18:50:48.762Z03/30 Read block 42 len 12288
25672023-12-10T18:50:48.762Z04/30 Read block 382 len 40960
25682023-12-10T18:50:48.762Z05/30 Write block 364 len 24576 data: 2 2 2 3 3 3
25692023-12-10T18:50:48.762Z06/30 Write block 157 len 40960 data: 2 2 2 3 4 4 4 3 3 3
25702023-12-10T18:50:48.762Z07/30 Write block 304 len 24576 data: 2 2 2 2 2 3
25712023-12-10T18:50:48.763Z08/30 Read block 318 len 28672
25722023-12-10T18:50:48.763Z09/30 Read block 18 len 36864
25732023-12-10T18:50:48.763Z10/30 Write block 500 len 32768 data: 3 3 3 3 3 3 4 3
25742023-12-10T18:50:48.763Z11/30 Write block 142 len 32768 data: 2 2 2 2 2 3 3 3
25752023-12-10T18:50:48.763Z12/30 Write block 524 len 4096 data: 4
25762023-12-10T18:50:48.764Z13/30 Read block 386 len 28672
25772023-12-10T18:50:48.764Z14/30 Read block 316 len 12288
25782023-12-10T18:50:48.764Z15/30 Write block 167 len 4096 data: 3
25792023-12-10T18:50:48.764Z16/30 Read block 523 len 36864
25802023-12-10T18:50:48.773Z17/30 Read block 231 len 12288
25812023-12-10T18:50:48.773Z18/30 Read block 135 len 8192
25822023-12-10T18:50:48.774Z19/30 Read block 28 len 12288
25832023-12-10T18:50:48.774Z20/30 Read block 298 len 12288
25842023-12-10T18:50:48.774Z21/30 Read block 520 len 12288
25852023-12-10T18:50:48.774Z22/30 Write block 238 len 36864 data: 5 6 6 5 6 6 6 6 5
25862023-12-10T18:50:48.774Z23/30 Read block 222 len 24576
25872023-12-10T18:50:48.783Z24/30 Read block 43 len 36864
25882023-12-10T18:50:48.783Z25/30 Read block 271 len 12288
25892023-12-10T18:50:48.784Z26/30 Read block 433 len 12288
25902023-12-10T18:50:48.784Z27/30 Read block 36 len 4096
25912023-12-10T18:50:48.784Z28/30 Write block 303 len 16384 data: 2 3 3 3
25922023-12-10T18:50:48.790Z29/30 Write block 575 len 28672 data: 4 4 3 3 3 3 3
25932023-12-10T18:50:48.790Z30/30 Write block 125 len 4096 data: 4
25942023-12-10T18:50:48.790Z----------------------------------------------------------------
25952023-12-10T18:50:48.791Z Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30
25962023-12-10T18:50:48.791ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
25972023-12-10T18:50:48.791Z 1 Acked 1000 Write 6 Done Done Done false
25982023-12-10T18:50:48.792Z 2 Acked 1001 Read 5 Done Done Done false
25992023-12-10T18:50:48.792Z 3 Acked 1002 Read 3 Done Sent Done false
26002023-12-10T18:50:48.793Z 4 Acked 1003 Read 10 Done Sent Done false
26012023-12-10T18:50:48.793Z 5 Acked 1004 Write 6 Done Sent Done false
26022023-12-10T18:50:48.793Z 6 Acked 1005 Write 10 Done Sent Done false
26032023-12-10T18:50:48.794Z 7 Acked 1006 Write 6 Done Sent Done false
26042023-12-10T18:50:48.794Z 8 Acked 1007 Read 7 Done Sent Done false
26052023-12-10T18:50:48.794Z 9 Acked 1008 Read 9 Done Sent Done false
26062023-12-10T18:50:48.795Z 10 Acked 1009 Write 8 Done Sent Done false
26072023-12-10T18:50:48.795Z 11 Acked 1010 Write 8 Done Sent Done false
26082023-12-10T18:50:48.795Z 12 Acked 1011 Write 1 Done Sent Done false
26092023-12-10T18:50:48.795Z 13 Acked 1012 Read 7 Done Sent Done false
26102023-12-10T18:50:48.796Z 14 Acked 1013 Read 3 Done Sent Done false
26112023-12-10T18:50:48.796Z 15 Acked 1014 Write 1 Done Sent Done false
26122023-12-10T18:50:48.796Z 16 Acked 1015 Read 9 Done Sent Done false
26132023-12-10T18:50:48.797Z 17 Acked 1016 Read 3 Done Sent Done false
26142023-12-10T18:50:48.797Z 18 Acked 1017 Read 2 Done Sent Done false
26152023-12-10T18:50:48.797Z 19 Acked 1018 Read 3 Done Sent Done false
26162023-12-10T18:50:48.797Z 20 Acked 1019 Read 3 Done Sent Done false
26172023-12-10T18:50:48.798Z 21 Acked 1020 Read 3 Done Sent Done false
26182023-12-10T18:50:48.798Z 22 Acked 1021 Write 9 Done Sent Done false
26192023-12-10T18:50:48.798Z 23 Acked 1022 Read 6 Done Sent Done false
26202023-12-10T18:50:48.798Z 24 Acked 1023 Read 9 Done Sent Done false
26212023-12-10T18:50:48.799Z 25 Acked 1024 Read 3 Done Sent Done false
26222023-12-10T18:50:48.799Z 26 Acked 1025 Read 3 Done Sent Done false
26232023-12-10T18:50:48.801Z 27 Acked 1026 Read 1 Done Sent Done false
26242023-12-10T18:50:48.801Z 28 Acked 1027 Write 4 Done Sent Done false
26252023-12-10T18:50:48.801Z 29 Acked 1028 Write 7 Sent Sent Sent false
26262023-12-10T18:50:48.802Z 30 Acked 1029 Write 1 Sent Sent Sent false
26272023-12-10T18:50:48.802Z STATES DS:0 DS:1 DS:2 TOTAL
26282023-12-10T18:50:48.802Z New 0 0 0 0
26292023-12-10T18:50:48.802Z Sent 2 28 2 32
26302023-12-10T18:50:48.803Z Done 28 2 28 58
26312023-12-10T18:50:48.803Z Skipped 0 0 0 0
26322023-12-10T18:50:48.803Z Error 0 0 0 0
26332023-12-10T18:50:48.804ZLast Flush: 0 0 0
26342023-12-10T18:50:48.804ZDownstairs last five completed:
26352023-12-10T18:50:48.804ZUpstairs last five completed: 30 29 28 27 26 25
26362023-12-10T18:50:48.804ZWrote out file "/tmp/test_repair_verify.data"
26372023-12-10T18:50:48.805Z
26382023-12-10T18:50:48.852ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
26392023-12-10T18:50:48.874ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
26402023-12-10T18:50:48.874Z 0 000-019 14 14 14 22 22 22 F F F
26412023-12-10T18:50:48.874Z 1 020-039 18 18 18 37 37 37 T T T
26422023-12-10T18:50:48.875Z 2 040-059 16 16 16 29 29 29 F F F
26432023-12-10T18:50:48.875Z 3 060-079 18 18 18 35 35 35 F F F
26442023-12-10T18:50:48.875Z 4 080-099 18 18 18 34 34 34 F F F
26452023-12-10T18:50:48.875Z 5 100-119 19 19 19 38 38 38 F F F
26462023-12-10T18:50:48.875Z 6 120-139 19 19 19 38 38 38 T F T <---
26472023-12-10T18:50:48.875Z 7 140-159 15 15 15 26 26 26 T F T <---
26482023-12-10T18:50:48.876Z 8 160-179 18 18 18 37 37 37 T F T <---
26492023-12-10T18:50:48.876Z 9 180-199 16 16 16 31 31 31 F F F
26502023-12-10T18:50:48.876Z 10 200-219 18 18 18 37 37 37 F F F
26512023-12-10T18:50:48.876Z 11 220-239 18 18 18 34 34 34 T F T <---
26522023-12-10T18:50:48.876Z 12 240-259 18 18 18 35 35 35 T F T <---
26532023-12-10T18:50:48.876Z 13 260-279 19 19 19 38 38 38 F F F
26542023-12-10T18:50:48.876Z 14 280-299 18 18 18 37 37 37 F F F
26552023-12-10T18:50:48.877Z 15 300-319 10 10 10 16 16 16 T F T <---
26562023-12-10T18:50:48.877Z 16 320-339 14 14 14 22 22 22 F F F
26572023-12-10T18:50:48.877Z 17 340-359 16 16 16 30 30 30 F F F
26582023-12-10T18:50:48.877Z 18 360-379 16 16 16 30 30 30 T F T <---
26592023-12-10T18:50:48.877Z 19 380-399 17 17 17 32 32 32 F F F
26602023-12-10T18:50:48.877Z 20 400-419 14 14 14 25 25 25 F F F
26612023-12-10T18:50:48.878Z 21 420-439 14 14 14 23 23 23 F F F
26622023-12-10T18:50:48.878Z 22 440-459 18 18 18 34 34 34 F F F
26632023-12-10T18:50:48.878Z 23 460-479 15 15 15 26 26 26 F F F
26642023-12-10T18:50:48.878Z 24 480-499 19 19 19 38 38 38 F F F
26652023-12-10T18:50:48.878Z 25 500-519 14 14 14 22 22 22 T F T <---
26662023-12-10T18:50:48.878Z 26 520-539 17 17 17 32 32 32 T F T <---
26672023-12-10T18:50:48.878Z 27 540-559 14 14 14 25 25 25 F F F
26682023-12-10T18:50:48.879Z 28 560-579 13 13 13 20 20 20 T F T <---
26692023-12-10T18:50:48.879Z 29 580-599 17 17 17 32 32 32 T F T <---
26702023-12-10T18:50:48.879ZMax gen: 19, Max flush: 38
26712023-12-10T18:50:48.879ZError: Difference in extent metadata found!
26722023-12-10T18:50:48.879ZA Difference in extent metadata is expected here
26732023-12-10T18:50:48.879ZOn loop 10
26742023-12-10T18:50:48.879Z
26752023-12-10T18:50:48.879ZVerifying data now
26762023-12-10T18:50:54.043ZNo change after verify
26772023-12-10T18:50:54.052ZLoop: 10 Downstairs dump after verify (and repair):
26782023-12-10T18:50:54.069ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
26792023-12-10T18:50:54.069Z 0 000-019 14 14 14 22 22 22 F F F
26802023-12-10T18:50:54.077Z 1 020-039 20 20 20 39 39 39 F F F
26812023-12-10T18:50:54.078Z 2 040-059 16 16 16 29 29 29 F F F
26822023-12-10T18:50:54.078Z 3 060-079 18 18 18 35 35 35 F F F
26832023-12-10T18:50:54.078Z 4 080-099 18 18 18 34 34 34 F F F
26842023-12-10T18:50:54.078Z 5 100-119 19 19 19 38 38 38 F F F
26852023-12-10T18:50:54.078Z 6 120-139 20 20 20 39 39 39 F F F
26862023-12-10T18:50:54.078Z 7 140-159 20 20 20 39 39 39 F F F
26872023-12-10T18:50:54.078Z 8 160-179 20 20 20 39 39 39 F F F
26882023-12-10T18:50:54.078Z 9 180-199 16 16 16 31 31 31 F F F
26892023-12-10T18:50:54.078Z 10 200-219 18 18 18 37 37 37 F F F
26902023-12-10T18:50:54.078Z 11 220-239 20 20 20 39 39 39 F F F
26912023-12-10T18:50:54.078Z 12 240-259 20 20 20 39 39 39 F F F
26922023-12-10T18:50:54.078Z 13 260-279 19 19 19 38 38 38 F F F
26932023-12-10T18:50:54.078Z 14 280-299 18 18 18 37 37 37 F F F
26942023-12-10T18:50:54.078Z 15 300-319 20 20 20 39 39 39 F F F
26952023-12-10T18:50:54.078Z 16 320-339 14 14 14 22 22 22 F F F
26962023-12-10T18:50:54.078Z 17 340-359 16 16 16 30 30 30 F F F
26972023-12-10T18:50:54.078Z 18 360-379 20 20 20 39 39 39 F F F
26982023-12-10T18:50:54.079Z 19 380-399 17 17 17 32 32 32 F F F
26992023-12-10T18:50:54.079Z 20 400-419 14 14 14 25 25 25 F F F
27002023-12-10T18:50:54.079Z 21 420-439 14 14 14 23 23 23 F F F
27012023-12-10T18:50:54.079Z 22 440-459 18 18 18 34 34 34 F F F
27022023-12-10T18:50:54.079Z 23 460-479 15 15 15 26 26 26 F F F
27032023-12-10T18:50:54.079Z 24 480-499 19 19 19 38 38 38 F F F
27042023-12-10T18:50:54.079Z 25 500-519 20 20 20 39 39 39 F F F
27052023-12-10T18:50:54.079Z 26 520-539 20 20 20 39 39 39 F F F
27062023-12-10T18:50:54.080Z 27 540-559 14 14 14 25 25 25 F F F
27072023-12-10T18:50:54.080Z 28 560-579 20 20 20 39 39 39 F F F
27082023-12-10T18:50:54.080Z 29 580-599 20 20 20 39 39 39 F F F
27092023-12-10T18:50:54.080ZMax gen: 20, Max flush: 39
27102023-12-10T18:50:54.080Z
27112023-12-10T18:50:54.080ZBegin loop 11
27122023-12-10T18:50:54.080ZDownstairs to restart: 0
27132023-12-10T18:50:54.103Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.875811222Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247}
27142023-12-10T18:50:54.103Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30Crucible runtime is spawned
27152023-12-10T18:50:54.103Z,"time":"2023-12-10T18:50:52.876424233Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247}
27162023-12-10T18:50:54.121Z{The guest has requested activation with gen:22
27172023-12-10T18:50:54.121Z"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.876657867Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247}
27182023-12-10T18:50:54.121Z{"msg":"Crucible 6fb3b630-7f99-4c3d-b018-84be99ff48de has session id: 435b6ac5-f75c-468f-ab1e-c2ca93993f69","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.876752959Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27192023-12-10T18:50:54.121Z{"msg":"Upstairs opts: Upstairs UUID: 6fb3b630-7f99-4c3d-b018-84be99ff48de, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.876944074Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27202023-12-10T18:50:54.121Z{"msg":"Crucible stats registered with UUID: 6fb3b630-7f99-4c3d-b018-84be99ff48de","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.877069664Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27212023-12-10T18:50:54.122Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.87716433Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69","task":"up_listen"}
27222023-12-10T18:50:54.122Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.8772442Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27232023-12-10T18:50:54.122Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.877289245Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27242023-12-10T18:50:54.122Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.877333372Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"looper":"0","session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27252023-12-10T18:50:54.122Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.877364462Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"looper":"2","session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27262023-12-10T18:50:54.122Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.877425541Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"looper":"1","session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27272023-12-10T18:50:54.122Z{"msg":"[2] 6fb3b630-7f99-4c3d-b018-84be99ff48de looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.877452708Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"looper":"2","session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27282023-12-10T18:50:54.122Z{"msg":"[0] 6fb3b630-7f99-4c3d-b018-84be99ff48de looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.877487977Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"looper":"0","session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27292023-12-10T18:50:54.122Z{"msg":"[1] 6fb3b630-7f99-4c3d-b018-84be99ff48de looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.877529118Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"looper":"1","session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27302023-12-10T18:50:54.122Z{"msg":"6fb3b630-7f99-4c3d-b018-84be99ff48de active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.877590903Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27312023-12-10T18:50:54.122Z{"msg":"Set desired generation to :22","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.877616517Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27322023-12-10T18:50:54.123Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.877672458Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27332023-12-10T18:50:54.123Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.877848803Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27342023-12-10T18:50:54.123Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.878084325Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27352023-12-10T18:50:54.124Z{"msg":"[2] 6fb3b630-7f99-4c3d-b018-84be99ff48de (435b6ac5-f75c-468f-ab1e-c2ca93993f69) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.878156496Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27362023-12-10T18:50:54.124Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.878291369Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27372023-12-10T18:50:54.124Z{"msg":"[2] client is_active_req TRUE, promote! session 435b6ac5-f75c-468f-ab1e-c2ca93993f69","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.878339874Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27382023-12-10T18:50:54.124Z{"msg":"[1] 6fb3b630-7f99-4c3d-b018-84be99ff48de (435b6ac5-f75c-468f-ab1e-c2ca93993f69) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.87848054Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27392023-12-10T18:50:54.124Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.878519954Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27402023-12-10T18:50:54.124Z{"msg":"[1] client is_active_req TRUE, promote! session 435b6ac5-f75c-468f-ab1e-c2ca93993f69","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.878544293Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27412023-12-10T18:50:54.124Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.879067908Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27422023-12-10T18:50:54.126Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.879160413Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27432023-12-10T18:50:54.126Z{"msg":"6fb3b630-7f99-4c3d-b018-84be99ff48de New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.879205003Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27442023-12-10T18:50:54.126Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.879312296Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27452023-12-10T18:50:54.126Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.879363266Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27462023-12-10T18:50:54.126Z{"msg":"6fb3b630-7f99-4c3d-b018-84be99ff48de New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.879407071Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27472023-12-10T18:50:54.126Z{"msg":"[2] 6fb3b630-7f99-4c3d-b018-84be99ff48de (435b6ac5-f75c-468f-ab1e-c2ca93993f69) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.879833605Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27482023-12-10T18:50:54.126Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.879909798Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27492023-12-10T18:50:54.126Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:52.879953066Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27502023-12-10T18:50:54.126Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.879992134Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27512023-12-10T18:50:54.126Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.880065655Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27522023-12-10T18:50:54.126Z{"msg":"6fb3b630-7f99-4c3d-b018-84be99ff48de New WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.880091494Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27532023-12-10T18:50:54.126Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.88011413Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27542023-12-10T18:50:54.126Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.880134619Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27552023-12-10T18:50:54.126Z{"msg":"[1] 6fb3b630-7f99-4c3d-b018-84be99ff48de (435b6ac5-f75c-468f-ab1e-c2ca93993f69) New WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.880239308Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27562023-12-10T18:50:54.126Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.880293828Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27572023-12-10T18:50:54.127Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:52.880317611Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27582023-12-10T18:50:54.127Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.880371488Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27592023-12-10T18:50:54.127Z{"msg":"[0] 6fb3b630-7f99-4c3d-b018-84be99ff48de (435b6ac5-f75c-468f-ab1e-c2ca93993f69) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.880408098Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27602023-12-10T18:50:54.127Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.880429713Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27612023-12-10T18:50:54.127Z{"msg":"[0] client is_active_req TRUE, promote! session 435b6ac5-f75c-468f-ab1e-c2ca93993f69","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.880452459Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27622023-12-10T18:50:54.128Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.880508542Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27632023-12-10T18:50:54.129Z{"msg":"6fb3b630-7f99-4c3d-b018-84be99ff48de WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.880571682Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27642023-12-10T18:50:54.129Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.880598345Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27652023-12-10T18:50:54.129Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.880639637Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27662023-12-10T18:50:54.129Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.881073583Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27672023-12-10T18:50:54.129Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.881140356Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27682023-12-10T18:50:54.129Z{"msg":"6fb3b630-7f99-4c3d-b018-84be99ff48de WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.881166514Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27692023-12-10T18:50:54.129Z{"msg":"[0] 6fb3b630-7f99-4c3d-b018-84be99ff48de (435b6ac5-f75c-468f-ab1e-c2ca93993f69) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.881692272Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27702023-12-10T18:50:54.129Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.881730823Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27712023-12-10T18:50:54.129Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:52.881752148Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27722023-12-10T18:50:54.130Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.881772228Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27732023-12-10T18:50:54.130Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.88181672Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27742023-12-10T18:50:54.130Z{"msg":"6fb3b630-7f99-4c3d-b018-84be99ff48de WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.881840991Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27752023-12-10T18:50:54.130Z{"msg":"[0]R flush_numbers[0..12]: [22, 39, 29, 35, 34, 38, 39, 39, 39, 31, 37, 39]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.881883494Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27762023-12-10T18:50:54.130Z{"msg":"[0]R generation[0..12]: [14, 20, 16, 18, 18, 19, 20, 20, 20, 16, 18, 20]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.881907015Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27772023-12-10T18:50:54.130Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.881929033Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27782023-12-10T18:50:54.130Z{"msg":"[1]R flush_numbers[0..12]: [22, 39, 29, 35, 34, 38, 39, 39, 39, 31, 37, 39]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.881969759Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27792023-12-10T18:50:54.130Z{"msg":"[1]R generation[0..12]: [14, 20, 16, 18, 18, 19, 20, 20, 20, 16, 18, 20]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.881993274Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27802023-12-10T18:50:54.130Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:22
27812023-12-10T18:50:54.130Z,"time":"2023-12-10T18:50:52.882049077Z","hostname":"Wait for a query_work_queue command to finish before sending IO
27822023-12-10T18:50:54.130Zip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27832023-12-10T18:50:54.131Z{"msg":"[2]R flush_numbers[0..12]: [22, 39, 29, 35, 34, 38, 39, 39, 39, 31, 37, 39]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.882076145Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27842023-12-10T18:50:54.131Z{"msg":"[2]R generation[0..12]: [14, 20, 16, 18, 18, 19, 20, 20, 20, 16, 18, 20]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.882121685Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27852023-12-10T18:50:54.131Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.882145295Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27862023-12-10T18:50:54.131Z{"msg":"Max found gen is 21","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.882166776Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27872023-12-10T18:50:54.131Z{"msg":"Generation requested: 22 >= found:21","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.882194513Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27882023-12-10T18:50:54.131Z{"msg":"Next flush: 40","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.882214996Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27892023-12-10T18:50:54.131Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.882243156Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27902023-12-10T18:50:54.131Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.882306897Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27912023-12-10T18:50:54.131Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.882346864Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27922023-12-10T18:50:54.131Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.882417825Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27932023-12-10T18:50:54.131Z{"msg":"6fb3b630-7f99-4c3d-b018-84be99ff48de is now active with session: 435b6ac5-f75c-468f-ab1e-c2ca93993f69","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.882464032Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27942023-12-10T18:50:54.131Z{"msg":"6fb3b630-7f99-4c3d-b018-84be99ff48de Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.882533321Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27952023-12-10T18:50:54.131Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.882575821Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27962023-12-10T18:50:54.131Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.882644511Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27972023-12-10T18:50:54.132Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.882718046Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27982023-12-10T18:50:54.132Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.882763939Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
27992023-12-10T18:50:54.132Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
28002023-12-10T18:50:54.132Zlevel":30,"time":"2023-12-10T18:50:52.882916982Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
28012023-12-10T18:50:54.132Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.882971898Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
28022023-12-10T18:50:54.132Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.88299565Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
28032023-12-10T18:50:54.132Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.883029229Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
28042023-12-10T18:50:54.132Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.883073314Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
28052023-12-10T18:50:54.134Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.883101179Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
28062023-12-10T18:50:54.134Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:52.883160429Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1247,"session_id":"435b6ac5-f75c-468f-ab1e-c2ca93993f69"}
28072023-12-10T18:50:54.142ZLoading write count information from file "/tmp/test_repair_verify.data"
28082023-12-10T18:50:54.142ZRun Repair workload
28092023-12-10T18:50:54.142Z01/30 Write block 165 len 40960 data: 4 4 4 2 3 3 4 4 4 4
28102023-12-10T18:50:54.142Z02/30 Read block 384 len 24576
28112023-12-10T18:50:54.142Z03/30 Read block 128 len 28672
28122023-12-10T18:50:54.142Z04/30 Read block 139 len 4096
28132023-12-10T18:50:54.142Z05/30 Read block 211 len 36864
28142023-12-10T18:50:54.143Z06/30 Flush
28152023-12-10T18:50:54.143Z07/30 Write block 107 len 28672 data: 3 3 3 3 2 2 2
28162023-12-10T18:50:54.143Z08/30 Read block 563 len 8192
28172023-12-10T18:50:54.143Z09/30 Write block 214 len 16384 data: 4 4 5 4
28182023-12-10T18:50:54.143Z10/30 Write block 372 len 4096 data: 2
28192023-12-10T18:50:54.143Z11/30 Read block 353 len 20480
28202023-12-10T18:50:54.143Z12/30 Read block 89 len 40960
28212023-12-10T18:50:54.145Z13/30 Read block 101 len 36864
28222023-12-10T18:50:54.145Z14/30 Read block 117 len 16384
28232023-12-10T18:50:54.145Z15/30 Read block 267 len 24576
28242023-12-10T18:50:54.151Z16/30 Read block 219 len 40960
28252023-12-10T18:50:54.151Z17/30 Read block 530 len 40960
28262023-12-10T18:50:54.151Z18/30 Read block 519 len 32768
28272023-12-10T18:50:54.151Z19/30 Write block 157 len 12288 data: 3 3 3
28282023-12-10T18:50:54.154Z20/30 Write block 575 len 36864 data: 5 5 4 4 4 4 4 2 2
28292023-12-10T18:50:54.154Z21/30 Read block 71 len 24576
28302023-12-10T18:50:54.154Z22/30 Read block 342 len 20480
28312023-12-10T18:50:54.157Z23/30 Write block 495 len 4096 data: 2
28322023-12-10T18:50:54.157Z24/30 Read block 176 len 24576
28332023-12-10T18:50:54.160Z25/30 Write block 270 len 40960 data: 3 3 3 4 5 5 5 4 4 4
28342023-12-10T18:50:54.160Z26/30 Write block 157 len 32768 data: 4 4 4 4 5 5 5 4
28352023-12-10T18:50:54.163Z27/30 Write block 351 len 40960 data: 4 5 5 2 2 3 4 3 3 3
28362023-12-10T18:50:54.163Z28/30 Read block 283 len 20480
28372023-12-10T18:50:54.166Z29/30 Write block 333 len 8192 data: 2 2
28382023-12-10T18:50:54.166Z30/30 Write block 161 len 32768 data: 6 6 6 5 5 5 5 3
28392023-12-10T18:50:54.170Z----------------------------------------------------------------
28402023-12-10T18:50:54.170Z Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:30
28412023-12-10T18:50:54.170ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
28422023-12-10T18:50:54.170Z 1 Acked 1000 Write 10 Done Done Done false
28432023-12-10T18:50:54.170Z 2 Acked 1001 Read 6 Done Done Done false
28442023-12-10T18:50:54.170Z 3 Acked 1002 Read 7 Done Done Done false
28452023-12-10T18:50:54.170Z 4 Acked 1003 Read 1 Done Done Done false
28462023-12-10T18:50:54.170Z 5 Acked 1004 Read 9 Done Done Done false
28472023-12-10T18:50:54.170Z 6 Acked 1005 Flush 0 Sent Done Done false
28482023-12-10T18:50:54.170Z 7 Acked 1006 Write 7 Sent Done Done false
28492023-12-10T18:50:54.170Z 8 Acked 1007 Read 2 Sent Done Done false
28502023-12-10T18:50:54.170Z 9 Acked 1008 Write 4 Sent Done Done false
28512023-12-10T18:50:54.170Z 10 Acked 1009 Write 1 Sent Done Done false
28522023-12-10T18:50:54.170Z 11 Acked 1010 Read 5 Sent Done Done false
28532023-12-10T18:50:54.170Z 12 Acked 1011 Read 10 Sent Done Done false
28542023-12-10T18:50:54.170Z 13 Acked 1012 Read 9 Sent Done Done false
28552023-12-10T18:50:54.170Z 14 Acked 1013 Read 4 Sent Done Done false
28562023-12-10T18:50:54.171Z 15 Acked 1014 Read 6 Sent Done Done false
28572023-12-10T18:50:54.171Z 16 Acked 1015 Read 10 Sent Done Done false
28582023-12-10T18:50:54.171Z 17 Acked 1016 Read 10 Sent Done Done false
28592023-12-10T18:50:54.171Z 18 Acked 1017 Read 8 Sent Done Done false
28602023-12-10T18:50:54.171Z 19 Acked 1018 Write 3 Sent Done Done false
28612023-12-10T18:50:54.171Z 20 Acked 1019 Write 9 Sent Done Done false
28622023-12-10T18:50:54.171Z 21 Acked 1020 Read 6 Sent Done Done false
28632023-12-10T18:50:54.171Z 22 Acked 1021 Read 5 Sent Done Done false
28642023-12-10T18:50:54.171Z 23 Acked 1022 Write 1 Sent Done Done false
28652023-12-10T18:50:54.171Z 24 Acked 1023 Read 6 Sent Done Done false
28662023-12-10T18:50:54.171Z 25 Acked 1024 Write 10 Sent Done Done false
28672023-12-10T18:50:54.171Z 26 Acked 1025 Write 8 Sent Done Done false
28682023-12-10T18:50:54.171Z 27 Acked 1026 Write 10 Sent Done Done false
28692023-12-10T18:50:54.171Z 28 Acked 1027 Read 5 Sent Done Done false
28702023-12-10T18:50:54.171Z 29 Acked 1028 Write 2 Sent Done Done false
28712023-12-10T18:50:54.171Z 30 Acked 1029 Write 8 Sent Sent Sent false
28722023-12-10T18:50:54.171Z STATES DS:0 DS:1 DS:2 TOTAL
28732023-12-10T18:50:54.171Z New 0 0 0 0
28742023-12-10T18:50:54.172Z Sent 25 1 1 27
28752023-12-10T18:50:54.172Z Done 5 29 29 63
28762023-12-10T18:50:54.172Z Skipped 0 0 0 0
28772023-12-10T18:50:54.172Z Error 0 0 0 0
28782023-12-10T18:50:54.172ZLast Flush: 0 1005 1005
28792023-12-10T18:50:54.172ZDownstairs last five completed:
28802023-12-10T18:50:54.172ZUpstairs last five completed: 30 29 28 27 26 25
28812023-12-10T18:50:54.173ZWrote out file "/tmp/test_repair_verify.data"
28822023-12-10T18:50:54.176Z
28832023-12-10T18:50:54.182ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
28842023-12-10T18:50:54.211ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
28852023-12-10T18:50:54.211Z 0 000-019 14 14 14 22 22 22 F F F
28862023-12-10T18:50:54.211Z 1 020-039 20 20 20 39 39 39 F F F
28872023-12-10T18:50:54.211Z 2 040-059 16 16 16 29 29 29 F F F
28882023-12-10T18:50:54.211Z 3 060-079 18 18 18 35 35 35 F F F
28892023-12-10T18:50:54.211Z 4 080-099 18 18 18 34 34 34 F F F
28902023-12-10T18:50:54.211Z 5 100-119 19 19 19 38 38 38 F T T <---
28912023-12-10T18:50:54.214Z 6 120-139 20 20 20 39 39 39 F F F
28922023-12-10T18:50:54.214Z 7 140-159 20 20 20 39 39 39 F T T <---
28932023-12-10T18:50:54.215Z 8 160-179 20 22 22 39 40 40 T T T <---
28942023-12-10T18:50:54.215Z 9 180-199 16 16 16 31 31 31 F F F
28952023-12-10T18:50:54.215Z 10 200-219 18 18 18 37 37 37 F T T <---
28962023-12-10T18:50:54.215Z 11 220-239 20 20 20 39 39 39 F F F
28972023-12-10T18:50:54.215Z 12 240-259 20 20 20 39 39 39 F F F
28982023-12-10T18:50:54.215Z 13 260-279 19 19 19 38 38 38 F T T <---
28992023-12-10T18:50:54.215Z 14 280-299 18 18 18 37 37 37 F F F
29002023-12-10T18:50:54.215Z 15 300-319 20 20 20 39 39 39 F F F
29012023-12-10T18:50:54.215Z 16 320-339 14 14 14 22 22 22 F T T <---
29022023-12-10T18:50:54.215Z 17 340-359 16 16 16 30 30 30 F T T <---
29032023-12-10T18:50:54.215Z 18 360-379 20 20 20 39 39 39 F T T <---
29042023-12-10T18:50:54.215Z 19 380-399 17 17 17 32 32 32 F F F
29052023-12-10T18:50:54.215Z 20 400-419 14 14 14 25 25 25 F F F
29062023-12-10T18:50:54.215Z 21 420-439 14 14 14 23 23 23 F F F
29072023-12-10T18:50:54.215Z 22 440-459 18 18 18 34 34 34 F F F
29082023-12-10T18:50:54.215Z 23 460-479 15 15 15 26 26 26 F F F
29092023-12-10T18:50:54.215Z 24 480-499 19 19 19 38 38 38 F T T <---
29102023-12-10T18:50:54.215Z 25 500-519 20 20 20 39 39 39 F F F
29112023-12-10T18:50:54.216Z 26 520-539 20 20 20 39 39 39 F F F
29122023-12-10T18:50:54.216Z 27 540-559 14 14 14 25 25 25 F F F
29132023-12-10T18:50:54.216Z 28 560-579 20 20 20 39 39 39 F T T <---
29142023-12-10T18:50:54.216Z 29 580-599 20 20 20 39 39 39 F T T <---
29152023-12-10T18:50:54.216ZMax gen: 22, Max flush: 40
29162023-12-10T18:50:54.216ZError: Difference in extent metadata found!
29172023-12-10T18:50:54.219ZA Difference in extent metadata is expected here
29182023-12-10T18:50:54.219ZOn loop 11
29192023-12-10T18:50:54.219Z
29202023-12-10T18:50:54.222ZVerifying data now
29212023-12-10T18:50:59.351ZNo change after verify
29222023-12-10T18:50:59.351ZLoop: 11 Downstairs dump after verify (and repair):
29232023-12-10T18:50:59.373ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
29242023-12-10T18:50:59.373Z 0 000-019 14 14 14 22 22 22 F F F
29252023-12-10T18:50:59.378Z 1 020-039 20 20 20 39 39 39 F F F
29262023-12-10T18:50:59.378Z 2 040-059 16 16 16 29 29 29 F F F
29272023-12-10T18:50:59.379Z 3 060-079 18 18 18 35 35 35 F F F
29282023-12-10T18:50:59.379Z 4 080-099 18 18 18 34 34 34 F F F
29292023-12-10T18:50:59.379Z 5 100-119 23 23 23 41 41 41 F F F
29302023-12-10T18:50:59.379Z 6 120-139 20 20 20 39 39 39 F F F
29312023-12-10T18:50:59.379Z 7 140-159 23 23 23 41 41 41 F F F
29322023-12-10T18:50:59.379Z 8 160-179 23 23 23 41 41 41 F F F
29332023-12-10T18:50:59.379Z 9 180-199 16 16 16 31 31 31 F F F
29342023-12-10T18:50:59.379Z 10 200-219 23 23 23 41 41 41 F F F
29352023-12-10T18:50:59.379Z 11 220-239 20 20 20 39 39 39 F F F
29362023-12-10T18:50:59.379Z 12 240-259 20 20 20 39 39 39 F F F
29372023-12-10T18:50:59.380Z 13 260-279 23 23 23 41 41 41 F F F
29382023-12-10T18:50:59.380Z 14 280-299 18 18 18 37 37 37 F F F
29392023-12-10T18:50:59.380Z 15 300-319 20 20 20 39 39 39 F F F
29402023-12-10T18:50:59.380Z 16 320-339 23 23 23 41 41 41 F F F
29412023-12-10T18:50:59.380Z 17 340-359 23 23 23 41 41 41 F F F
29422023-12-10T18:50:59.380Z 18 360-379 23 23 23 41 41 41 F F F
29432023-12-10T18:50:59.380Z 19 380-399 17 17 17 32 32 32 F F F
29442023-12-10T18:50:59.380Z 20 400-419 14 14 14 25 25 25 F F F
29452023-12-10T18:50:59.380Z 21 420-439 14 14 14 23 23 23 F F F
29462023-12-10T18:50:59.380Z 22 440-459 18 18 18 34 34 34 F F F
29472023-12-10T18:50:59.381Z 23 460-479 15 15 15 26 26 26 F F F
29482023-12-10T18:50:59.381Z 24 480-499 23 23 23 41 41 41 F F F
29492023-12-10T18:50:59.381Z 25 500-519 20 20 20 39 39 39 F F F
29502023-12-10T18:50:59.382Z 26 520-539 20 20 20 39 39 39 F F F
29512023-12-10T18:50:59.382Z 27 540-559 14 14 14 25 25 25 F F F
29522023-12-10T18:50:59.382Z 28 560-579 23 23 23 41 41 41 F F F
29532023-12-10T18:50:59.382Z 29 580-599 23 23 23 41 41 41 F F F
29542023-12-10T18:50:59.382ZMax gen: 23, Max flush: 41
29552023-12-10T18:50:59.382Z
29562023-12-10T18:50:59.382ZBegin loop 12
29572023-12-10T18:50:59.382ZDownstairs to restart: 1
29582023-12-10T18:50:59.408Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.183048613Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255}
29592023-12-10T18:50:59.408Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.183452314Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255}
29602023-12-10T18:50:59.413Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.183504835Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255}
29612023-12-10T18:50:59.413Z{"msg":"Crucible be29c87a-29ba-49f8-aba2-c292c5d7a265 has session id: bd4082a7-2b72-47d4-a6dc-d738d4cdccd0","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.183570218Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29622023-12-10T18:50:59.413Z{"msg":"Upstairs opts: Upstairs UUID: be29c87a-29ba-49f8-aba2-c292c5d7a265, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.183665958Z","hostname":"Crucible runtime is spawned
29632023-12-10T18:50:59.413Zip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29642023-12-10T18:50:59.413Z{"msg":"Crucible stats registered with UUID: be29c87a-29ba-49f8-aba2-c292c5d7a265","v":0,"name":"crucible","level":30The guest has requested activation with gen:24
29652023-12-10T18:50:59.414Z,"time":"2023-12-10T18:50:58.183827831Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29662023-12-10T18:50:59.414Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.183960187Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0","task":"up_listen"}
29672023-12-10T18:50:59.414Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.184047704Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29682023-12-10T18:50:59.414Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.184333054Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29692023-12-10T18:50:59.414Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.184372418Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"looper":"2","session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29702023-12-10T18:50:59.414Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.184399092Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"looper":"1","session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29712023-12-10T18:50:59.414Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.184446744Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"looper":"0","session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29722023-12-10T18:50:59.414Z{"msg":"[1] be29c87a-29ba-49f8-aba2-c292c5d7a265 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.184472177Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"looper":"1","session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29732023-12-10T18:50:59.415Z{"msg":"[0] be29c87a-29ba-49f8-aba2-c292c5d7a265 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.184508233Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"looper":"0","session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29742023-12-10T18:50:59.415Z{"msg":"[2] be29c87a-29ba-49f8-aba2-c292c5d7a265 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.184546055Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"looper":"2","session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29752023-12-10T18:50:59.415Z{"msg":"be29c87a-29ba-49f8-aba2-c292c5d7a265 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.184594809Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29762023-12-10T18:50:59.415Z{"msg":"Set desired generation to :24","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.184617952Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29772023-12-10T18:50:59.415Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.184659155Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29782023-12-10T18:50:59.416Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.184923624Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29792023-12-10T18:50:59.416Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.185006599Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29802023-12-10T18:50:59.416Z{"msg":"[0] be29c87a-29ba-49f8-aba2-c292c5d7a265 (bd4082a7-2b72-47d4-a6dc-d738d4cdccd0) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.185393764Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29812023-12-10T18:50:59.416Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.185491345Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29822023-12-10T18:50:59.416Z{"msg":"[0] client is_active_req TRUE, promote! session bd4082a7-2b72-47d4-a6dc-d738d4cdccd0","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.185533537Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29832023-12-10T18:50:59.416Z{"msg":"[2] be29c87a-29ba-49f8-aba2-c292c5d7a265 (bd4082a7-2b72-47d4-a6dc-d738d4cdccd0) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.185643854Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29842023-12-10T18:50:59.416Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.1856916Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29852023-12-10T18:50:59.416Z{"msg":"[2] client is_active_req TRUE, promote! session bd4082a7-2b72-47d4-a6dc-d738d4cdccd0","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.185771393Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29862023-12-10T18:50:59.417Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.186112498Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29872023-12-10T18:50:59.417Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.186172062Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29882023-12-10T18:50:59.417Z{"msg":"be29c87a-29ba-49f8-aba2-c292c5d7a265 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.186195988Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29892023-12-10T18:50:59.417Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.186313565Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29902023-12-10T18:50:59.417Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.18641903Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29912023-12-10T18:50:59.417Z{"msg":"be29c87a-29ba-49f8-aba2-c292c5d7a265 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.186492057Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29922023-12-10T18:50:59.417Z{"msg":"[1] be29c87a-29ba-49f8-aba2-c292c5d7a265 (bd4082a7-2b72-47d4-a6dc-d738d4cdccd0) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.186521959Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29932023-12-10T18:50:59.417Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.186554385Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29942023-12-10T18:50:59.417Z{"msg":"[1] client is_active_req TRUE, promote! session bd4082a7-2b72-47d4-a6dc-d738d4cdccd0","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.186608603Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29952023-12-10T18:50:59.424Z{"msg":"[0] be29c87a-29ba-49f8-aba2-c292c5d7a265 (bd4082a7-2b72-47d4-a6dc-d738d4cdccd0) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.187111648Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29962023-12-10T18:50:59.424Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.187225082Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29972023-12-10T18:50:59.424Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:58.187249506Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29982023-12-10T18:50:59.424Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.187307065Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
29992023-12-10T18:50:59.424Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.18733636Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30002023-12-10T18:50:59.424Z{"msg":"be29c87a-29ba-49f8-aba2-c292c5d7a265 WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.1874014Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30012023-12-10T18:50:59.424Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.187430641Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30022023-12-10T18:50:59.424Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.187499109Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30032023-12-10T18:50:59.424Z{"msg":"[2] be29c87a-29ba-49f8-aba2-c292c5d7a265 (bd4082a7-2b72-47d4-a6dc-d738d4cdccd0) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.187526301Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30042023-12-10T18:50:59.424Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.18758349Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30052023-12-10T18:50:59.427Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:58.187611741Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30062023-12-10T18:50:59.427Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.187660883Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30072023-12-10T18:50:59.427Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.18768464Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30082023-12-10T18:50:59.427Z{"msg":"be29c87a-29ba-49f8-aba2-c292c5d7a265 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.187741189Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30092023-12-10T18:50:59.427Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.187770796Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30102023-12-10T18:50:59.427Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.187823634Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30112023-12-10T18:50:59.427Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.187848869Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30122023-12-10T18:50:59.427Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.187883838Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30132023-12-10T18:50:59.427Z{"msg":"be29c87a-29ba-49f8-aba2-c292c5d7a265 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.187950313Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30142023-12-10T18:50:59.427Z{"msg":"[1] be29c87a-29ba-49f8-aba2-c292c5d7a265 (bd4082a7-2b72-47d4-a6dc-d738d4cdccd0) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.188625932Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30152023-12-10T18:50:59.427Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.188710225Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30162023-12-10T18:50:59.427Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:50:58.188751268Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30172023-12-10T18:50:59.427Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.18881647Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30182023-12-10T18:50:59.428Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.188883811Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30192023-12-10T18:50:59.428Z{"msg":"be29c87a-29ba-49f8-aba2-c292c5d7a265 WaitQuorum WaitQuorum WaitQuorum","The guest has finished waiting for activation with:24
30202023-12-10T18:50:59.428ZWait for a query_work_queue command to finish before sending IO
30212023-12-10T18:50:59.428Zv":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.188973848Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30222023-12-10T18:50:59.428Z{"msg":"[0]R flush_numbers[0..12]: [22, 39, 29, 35, 34, 41, 39, 41, 41, 31, 41, 39]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.189066645Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30232023-12-10T18:50:59.428Z{"msg":"[0]R generation[0..12]: [14, 20, 16, 18, 18, 23, 20, 23, 23, 16, 23, 20]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.189163919Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30242023-12-10T18:50:59.428Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.189209563Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30252023-12-10T18:50:59.428Z{"msg":"[1]R flush_numbers[0..12]: [22, 39, 29, 35, 34, 41, 39, 41, 41, 31, 41, 39]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.189278276Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30262023-12-10T18:50:59.428Z{"msg":"[1]R generation[0..12]: [14, 20, 16, 18, 18, 23, 20, 23, 23, 16, 23, 20]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.189355976Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30272023-12-10T18:50:59.428Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.189427777Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30282023-12-10T18:50:59.428Z{"msg":"[2]R flush_numbers[0..12]: [22, 39, 29, 35, 34, 41, 39, 41, 41, 31, 41, 39]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.189453627Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30292023-12-10T18:50:59.428Z{"msg":"[2]R generation[0..12]: [14, 20, 16, 18, 18, 23, 20, 23, 23, 16, 23, 20]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.189495693Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30302023-12-10T18:50:59.428Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.189519712Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30312023-12-10T18:50:59.428Z{"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.18953938Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30322023-12-10T18:50:59.428Z{"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.1895809Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30332023-12-10T18:50:59.428Z{"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.189602775Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
30342023-12-10T18:50:59.428Zbd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30352023-12-10T18:50:59.429Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.189734129Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30362023-12-10T18:50:59.429Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.18976088Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30372023-12-10T18:50:59.429Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.189781388Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30382023-12-10T18:50:59.429Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.189821855Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30392023-12-10T18:50:59.429Z{"msg":"be29c87a-29ba-49f8-aba2-c292c5d7a265 is now active with session: bd4082a7-2b72-47d4-a6dc-d738d4cdccd0","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.189845792Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30402023-12-10T18:50:59.429Z{"msg":"be29c87a-29ba-49f8-aba2-c292c5d7a265 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.189886568Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30412023-12-10T18:50:59.429Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.189909051Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30422023-12-10T18:50:59.429Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.189929965Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30432023-12-10T18:50:59.429Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.189949711Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30442023-12-10T18:50:59.430Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.189969439Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30452023-12-10T18:50:59.430Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.19002422Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30462023-12-10T18:50:59.430Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.190048615Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30472023-12-10T18:50:59.430Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.190090054Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30482023-12-10T18:50:59.431Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.190112005Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30492023-12-10T18:50:59.431Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.190151451Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":Loading write count information from file "/tmp/test_repair_verify.data"
30502023-12-10T18:50:59.431Z1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30512023-12-10T18:50:59.431Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.190180817Z"Run Repair workload
30522023-12-10T18:50:59.431Z,"hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30532023-12-10T18:50:59.431Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:50:58.190203882Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1255,"session_id":"bd4082a7-2b72-47d4-a6dc-d738d4cdccd0"}
30542023-12-10T18:50:59.431Z01/30 Write block 146 len 20480 data: 3 4 4 4 3
30552023-12-10T18:50:59.431Z02/30 Flush
30562023-12-10T18:50:59.431Z03/30 Write block 360 len 16384 data: 4 2 2 2
30572023-12-10T18:50:59.431Z04/30 Read block 262 len 8192
30582023-12-10T18:50:59.431Z05/30 Read block 588 len 16384
30592023-12-10T18:50:59.435Z06/30 Read block 288 len 40960
30602023-12-10T18:50:59.435Z07/30 Write block 213 len 24576 data: 3 5 5 6 5 4
30612023-12-10T18:50:59.435Z08/30 Read block 480 len 40960
30622023-12-10T18:50:59.435Z09/30 Read block 289 len 8192
30632023-12-10T18:50:59.435Z10/30 Write block 17 len 40960 data: 3 2 2 2 3 4 4 4 4 3
30642023-12-10T18:50:59.435Z11/30 Read block 552 len 20480
30652023-12-10T18:50:59.438Z12/30 Write block 157 len 16384 data: 5 5 5 5
30662023-12-10T18:50:59.438Z13/30 Write block 107 len 28672 data: 4 4 4 4 3 3 3
30672023-12-10T18:50:59.441Z14/30 Read block 510 len 20480
30682023-12-10T18:50:59.441Z15/30 Write block 30 len 12288 data: 2 3 3
30692023-12-10T18:50:59.444Z16/30 Read block 326 len 4096
30702023-12-10T18:50:59.444Z17/30 Read block 408 len 20480
30712023-12-10T18:50:59.447Z18/30 Write block 45 len 4096 data: 3
30722023-12-10T18:50:59.447Z19/30 Read block 59 len 20480
30732023-12-10T18:50:59.447Z20/30 Read block 400 len 16384
30742023-12-10T18:50:59.450Z21/30 Read block 362 len 8192
30752023-12-10T18:50:59.450Z22/30 Read block 77 len 40960
30762023-12-10T18:50:59.453Z23/30 Write block 35 len 32768 data: 3 4 4 3 3 3 3 3
30772023-12-10T18:50:59.460Z24/30 Write block 241 len 28672 data: 6 7 7 7 7 6 5
30782023-12-10T18:50:59.460Z25/30 Write block 459 len 12288 data: 3 3 3
30792023-12-10T18:50:59.460Z26/30 Flush
30802023-12-10T18:50:59.464Z27/30 Write block 144 len 12288 data: 3 3 4
30812023-12-10T18:50:59.464Z28/30 Write block 217 len 28672 data: 6 5 3 3 4 4 3
30822023-12-10T18:50:59.464Z29/30 Write block 239 len 4096 data: 7
30832023-12-10T18:50:59.467Z30/30 Read block 53 len 36864
30842023-12-10T18:50:59.471Z----------------------------------------------------------------
30852023-12-10T18:50:59.471Z Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:30
30862023-12-10T18:50:59.471ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
30872023-12-10T18:50:59.471Z 1 Acked 1000 Write 5 Done Sent Done false
30882023-12-10T18:50:59.471Z 2 Acked 1001 Flush 0 Done Sent Done false
30892023-12-10T18:50:59.471Z 3 Acked 1002 Write 4 Done Sent Done false
30902023-12-10T18:50:59.471Z 4 Acked 1003 Read 2 Done Sent Done false
30912023-12-10T18:50:59.471Z 5 Acked 1004 Read 4 Done Sent Done false
30922023-12-10T18:50:59.471Z 6 Acked 1005 Read 10 Done Sent Done false
30932023-12-10T18:50:59.472Z 7 Acked 1006 Write 6 Done Sent Done false
30942023-12-10T18:50:59.472Z 8 Acked 1007 Read 10 Done Sent Done false
30952023-12-10T18:50:59.472Z 9 Acked 1008 Read 2 Done Sent Done false
30962023-12-10T18:50:59.472Z 10 Acked 1009 Write 10 Done Sent Done false
30972023-12-10T18:50:59.472Z 11 Acked 1010 Read 5 Done Sent Done false
30982023-12-10T18:50:59.472Z 12 Acked 1011 Write 4 Done Sent Done false
30992023-12-10T18:50:59.472Z 13 Acked 1012 Write 7 Done Sent Done false
31002023-12-10T18:50:59.472Z 14 Acked 1013 Read 5 Done Sent Done false
31012023-12-10T18:50:59.472Z 15 Acked 1014 Write 3 Done Sent Done false
31022023-12-10T18:50:59.472Z 16 Acked 1015 Read 1 Done Sent Done false
31032023-12-10T18:50:59.472Z 17 Acked 1016 Read 5 Done Sent Done false
31042023-12-10T18:50:59.472Z 18 Acked 1017 Write 1 Done Sent Done false
31052023-12-10T18:50:59.472Z 19 Acked 1018 Read 5 Done Sent Done false
31062023-12-10T18:50:59.473Z 20 Acked 1019 Read 4 Done Sent Done false
31072023-12-10T18:50:59.473Z 21 Acked 1020 Read 2 Done Sent Done false
31082023-12-10T18:50:59.473Z 22 Acked 1021 Read 10 Done Sent Done false
31092023-12-10T18:50:59.473Z 23 Acked 1022 Write 8 Done Sent Done false
31102023-12-10T18:50:59.473Z 24 Acked 1023 Write 7 Done Sent Done false
31112023-12-10T18:50:59.473Z 25 Acked 1024 Write 3 Done Sent Done false
31122023-12-10T18:50:59.473Z 26 Acked 1025 Flush 0 Done Sent Done false
31132023-12-10T18:50:59.473Z 27 Acked 1026 Write 3 Done Sent Done false
31142023-12-10T18:50:59.473Z 28 Acked 1027 Write 7 Done Sent Done false
31152023-12-10T18:50:59.473Z 29 Acked 1028 Write 1 Done Sent Done false
31162023-12-10T18:50:59.475Z 30 Acked 1029 Read 9 Done Sent Sent false
31172023-12-10T18:50:59.475Z STATES DS:0 DS:1 DS:2 TOTAL
31182023-12-10T18:50:59.476Z New 0 0 0 0
31192023-12-10T18:50:59.476Z Sent 0 30 1 31
31202023-12-10T18:50:59.476Z Done 30 0 29 59
31212023-12-10T18:50:59.476Z Skipped 0 0 0 0
31222023-12-10T18:50:59.476Z Error 0 0 0 0
31232023-12-10T18:50:59.476ZLast Flush: 1025 0 1025
31242023-12-10T18:50:59.476ZDownstairs last five completed:
31252023-12-10T18:50:59.476ZUpstairs last five completed: 30 29 28 27 26 25
31262023-12-10T18:50:59.477ZWrote out file "/tmp/test_repair_verify.data"
31272023-12-10T18:50:59.480Z
31282023-12-10T18:50:59.480ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
31292023-12-10T18:50:59.502ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
31302023-12-10T18:50:59.502Z 0 000-019 24 14 24 43 22 43 F F F <---
31312023-12-10T18:50:59.502Z 1 020-039 24 20 24 43 39 43 F F F <---
31322023-12-10T18:50:59.502Z 2 040-059 24 16 24 43 29 43 F F F <---
31332023-12-10T18:50:59.503Z 3 060-079 18 18 18 35 35 35 F F F
31342023-12-10T18:50:59.503Z 4 080-099 18 18 18 34 34 34 F F F
31352023-12-10T18:50:59.503Z 5 100-119 24 23 24 43 41 43 F F F <---
31362023-12-10T18:50:59.503Z 6 120-139 20 20 20 39 39 39 F F F
31372023-12-10T18:50:59.503Z 7 140-159 24 23 24 43 41 43 T F T <---
31382023-12-10T18:50:59.509Z 8 160-179 24 23 24 43 41 43 F F F <---
31392023-12-10T18:50:59.509Z 9 180-199 16 16 16 31 31 31 F F F
31402023-12-10T18:50:59.509Z 10 200-219 24 23 24 43 41 43 T F T <---
31412023-12-10T18:50:59.510Z 11 220-239 20 20 20 39 39 39 T F T <---
31422023-12-10T18:50:59.510Z 12 240-259 24 20 24 43 39 43 F F F <---
31432023-12-10T18:50:59.510Z 13 260-279 23 23 23 41 41 41 F F F
31442023-12-10T18:50:59.510Z 14 280-299 18 18 18 37 37 37 F F F
31452023-12-10T18:50:59.510Z 15 300-319 20 20 20 39 39 39 F F F
31462023-12-10T18:50:59.510Z 16 320-339 23 23 23 41 41 41 F F F
31472023-12-10T18:50:59.510Z 17 340-359 23 23 23 41 41 41 F F F
31482023-12-10T18:50:59.510Z 18 360-379 24 23 24 43 41 43 F F F <---
31492023-12-10T18:50:59.510Z 19 380-399 17 17 17 32 32 32 F F F
31502023-12-10T18:50:59.510Z 20 400-419 14 14 14 25 25 25 F F F
31512023-12-10T18:50:59.510Z 21 420-439 14 14 14 23 23 23 F F F
31522023-12-10T18:50:59.510Z 22 440-459 24 18 24 43 34 43 F F F <---
31532023-12-10T18:50:59.510Z 23 460-479 24 15 24 43 26 43 F F F <---
31542023-12-10T18:50:59.510Z 24 480-499 23 23 23 41 41 41 F F F
31552023-12-10T18:50:59.510Z 25 500-519 20 20 20 39 39 39 F F F
31562023-12-10T18:50:59.510Z 26 520-539 20 20 20 39 39 39 F F F
31572023-12-10T18:50:59.511Z 27 540-559 14 14 14 25 25 25 F F F
31582023-12-10T18:50:59.511Z 28 560-579 23 23 23 41 41 41 F F F
31592023-12-10T18:50:59.511Z 29 580-599 23 23 23 41 41 41 F F F
31602023-12-10T18:50:59.511ZMax gen: 24, Max flush: 43
31612023-12-10T18:50:59.511ZError: Difference in extent metadata found!
31622023-12-10T18:50:59.514ZA Difference in extent metadata is expected here
31632023-12-10T18:50:59.514ZOn loop 12
31642023-12-10T18:50:59.514Z
31652023-12-10T18:50:59.514ZVerifying data now
31662023-12-10T18:51:04.690ZNo change after verify
31672023-12-10T18:51:04.690ZLoop: 12 Downstairs dump after verify (and repair):
31682023-12-10T18:51:04.711ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
31692023-12-10T18:51:04.711Z 0 000-019 24 24 24 43 43 43 F F F
31702023-12-10T18:51:04.716Z 1 020-039 24 24 24 43 43 43 F F F
31712023-12-10T18:51:04.716Z 2 040-059 24 24 24 43 43 43 F F F
31722023-12-10T18:51:04.717Z 3 060-079 18 18 18 35 35 35 F F F
31732023-12-10T18:51:04.717Z 4 080-099 18 18 18 34 34 34 F F F
31742023-12-10T18:51:04.717Z 5 100-119 24 24 24 43 43 43 F F F
31752023-12-10T18:51:04.717Z 6 120-139 20 20 20 39 39 39 F F F
31762023-12-10T18:51:04.717Z 7 140-159 25 25 25 44 44 44 F F F
31772023-12-10T18:51:04.717Z 8 160-179 24 24 24 43 43 43 F F F
31782023-12-10T18:51:04.717Z 9 180-199 16 16 16 31 31 31 F F F
31792023-12-10T18:51:04.717Z 10 200-219 25 25 25 44 44 44 F F F
31802023-12-10T18:51:04.717Z 11 220-239 25 25 25 44 44 44 F F F
31812023-12-10T18:51:04.717Z 12 240-259 24 24 24 43 43 43 F F F
31822023-12-10T18:51:04.717Z 13 260-279 23 23 23 41 41 41 F F F
31832023-12-10T18:51:04.717Z 14 280-299 18 18 18 37 37 37 F F F
31842023-12-10T18:51:04.717Z 15 300-319 20 20 20 39 39 39 F F F
31852023-12-10T18:51:04.717Z 16 320-339 23 23 23 41 41 41 F F F
31862023-12-10T18:51:04.717Z 17 340-359 23 23 23 41 41 41 F F F
31872023-12-10T18:51:04.717Z 18 360-379 24 24 24 43 43 43 F F F
31882023-12-10T18:51:04.718Z 19 380-399 17 17 17 32 32 32 F F F
31892023-12-10T18:51:04.718Z 20 400-419 14 14 14 25 25 25 F F F
31902023-12-10T18:51:04.718Z 21 420-439 14 14 14 23 23 23 F F F
31912023-12-10T18:51:04.718Z 22 440-459 24 24 24 43 43 43 F F F
31922023-12-10T18:51:04.718Z 23 460-479 24 24 24 43 43 43 F F F
31932023-12-10T18:51:04.718Z 24 480-499 23 23 23 41 41 41 F F F
31942023-12-10T18:51:04.718Z 25 500-519 20 20 20 39 39 39 F F F
31952023-12-10T18:51:04.718Z 26 520-539 20 20 20 39 39 39 F F F
31962023-12-10T18:51:04.718Z 27 540-559 14 14 14 25 25 25 F F F
31972023-12-10T18:51:04.718Z 28 560-579 23 23 23 41 41 41 F F F
31982023-12-10T18:51:04.718Z 29 580-599 23 23 23 41 41 41 F F F
31992023-12-10T18:51:04.720ZMax gen: 25, Max flush: 44
32002023-12-10T18:51:04.720Z
32012023-12-10T18:51:04.720ZBegin loop 13
32022023-12-10T18:51:04.721ZDownstairs to restart: 1
32032023-12-10T18:51:04.745Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.519730525Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263}
32042023-12-10T18:51:04.745Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.520186036Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263}
32052023-12-10T18:51:04.745Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.520245059Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263}
32062023-12-10T18:51:04.763Z{"msg":"Crucible 07c5c247-25ca-4706-bb90-1152c1d3873c has session id: 608c8f54-17b4-4e5e-b0a4-8a056e96aec8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.520307236Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32072023-12-10T18:51:04.763Z{"msg":"Upstairs opts: Upstairs UUID: 07c5c247-25ca-4706-bb90-1152c1d3873c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.520526108Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32082023-12-10T18:51:04.763Z{"msg":"Crucible stats registered with UUID: 07c5c247-25ca-4706-bb90-1152c1d3873c","v":0,"name":"Crucible runtime is spawned
32092023-12-10T18:51:04.763Zcrucible","level":30,"time":"2023-12-10T18:51:03.5206623Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32102023-12-10T18:51:04.763Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30The guest has requested activation with gen:26
32112023-12-10T18:51:04.763Z,"time":"2023-12-10T18:51:03.520767729Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"looper":"0","session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32122023-12-10T18:51:04.763Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.520869545Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8","task":"up_listen"}
32132023-12-10T18:51:04.764Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.52103389Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32142023-12-10T18:51:04.764Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.521078823Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32152023-12-10T18:51:04.764Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.521149088Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"looper":"1","session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32162023-12-10T18:51:04.764Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.521176149Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"looper":"2","session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32172023-12-10T18:51:04.765Z{"msg":"[0] 07c5c247-25ca-4706-bb90-1152c1d3873c looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.521198066Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"looper":"0","session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32182023-12-10T18:51:04.765Z{"msg":"[1] 07c5c247-25ca-4706-bb90-1152c1d3873c looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.521241297Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"looper":"1","session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32192023-12-10T18:51:04.765Z{"msg":"[2] 07c5c247-25ca-4706-bb90-1152c1d3873c looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.521272047Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"looper":"2","session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32202023-12-10T18:51:04.765Z{"msg":"07c5c247-25ca-4706-bb90-1152c1d3873c active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.521532518Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32212023-12-10T18:51:04.765Z{"msg":"Set desired generation to :26","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.521584492Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32222023-12-10T18:51:04.765Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.521710733Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32232023-12-10T18:51:04.765Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.522061416Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32242023-12-10T18:51:04.765Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.522117254Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32252023-12-10T18:51:04.765Z{"msg":"[0] 07c5c247-25ca-4706-bb90-1152c1d3873c (608c8f54-17b4-4e5e-b0a4-8a056e96aec8) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.522429662Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32262023-12-10T18:51:04.765Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.522506059Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32272023-12-10T18:51:04.765Z{"msg":"[0] client is_active_req TRUE, promote! session 608c8f54-17b4-4e5e-b0a4-8a056e96aec8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.522550216Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32282023-12-10T18:51:04.765Z{"msg":"[2] 07c5c247-25ca-4706-bb90-1152c1d3873c (608c8f54-17b4-4e5e-b0a4-8a056e96aec8) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.522633212Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32292023-12-10T18:51:04.765Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.522709757Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32302023-12-10T18:51:04.765Z{"msg":"[2] client is_active_req TRUE, promote! session 608c8f54-17b4-4e5e-b0a4-8a056e96aec8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.522754471Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32312023-12-10T18:51:04.765Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.523232982Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32322023-12-10T18:51:04.765Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.523303497Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32332023-12-10T18:51:04.765Z{"msg":"07c5c247-25ca-4706-bb90-1152c1d3873c WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.523417878Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32342023-12-10T18:51:04.766Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.523463782Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32352023-12-10T18:51:04.766Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.523506609Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32362023-12-10T18:51:04.766Z{"msg":"07c5c247-25ca-4706-bb90-1152c1d3873c WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.523593031Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32372023-12-10T18:51:04.766Z{"msg":"[1] 07c5c247-25ca-4706-bb90-1152c1d3873c (608c8f54-17b4-4e5e-b0a4-8a056e96aec8) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.523642447Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32382023-12-10T18:51:04.766Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.523713697Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32392023-12-10T18:51:04.766Z{"msg":"[1] client is_active_req TRUE, promote! session 608c8f54-17b4-4e5e-b0a4-8a056e96aec8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.523756768Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32402023-12-10T18:51:04.766Z{"msg":"[0] 07c5c247-25ca-4706-bb90-1152c1d3873c (608c8f54-17b4-4e5e-b0a4-8a056e96aec8) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.524077713Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32412023-12-10T18:51:04.766Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.524246116Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32422023-12-10T18:51:04.767Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:03.524292171Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32432023-12-10T18:51:04.767Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.524331387Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32442023-12-10T18:51:04.767Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.524368229Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32452023-12-10T18:51:04.767Z{"msg":"07c5c247-25ca-4706-bb90-1152c1d3873c WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.524425108Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32462023-12-10T18:51:04.767Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.5244649Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32472023-12-10T18:51:04.767Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.524524579Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32482023-12-10T18:51:04.767Z{"msg":"[2] 07c5c247-25ca-4706-bb90-1152c1d3873c (608c8f54-17b4-4e5e-b0a4-8a056e96aec8) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.524569691Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32492023-12-10T18:51:04.767Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.524625315Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32502023-12-10T18:51:04.767Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:03.524665462Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32512023-12-10T18:51:04.767Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.524735559Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32522023-12-10T18:51:04.767Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.52477594Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32532023-12-10T18:51:04.767Z{"msg":"07c5c247-25ca-4706-bb90-1152c1d3873c WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.524833842Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32542023-12-10T18:51:04.768Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.524874003Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32552023-12-10T18:51:04.768Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.524928687Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32562023-12-10T18:51:04.768Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.5249706Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32572023-12-10T18:51:04.768Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.525042024Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32582023-12-10T18:51:04.768Z{"msg":"07c5c247-25ca-4706-bb90-1152c1d3873c WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.525084254Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32592023-12-10T18:51:04.796Z{"msg":"[1] 07c5c247-25ca-4706-bb90-1152c1d3873c (608c8f54-17b4-4e5e-b0a4-8a056e96aec8) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.525414638Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32602023-12-10T18:51:04.796Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.525480024Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32612023-12-10T18:51:04.796Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:03.525520957Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32622023-12-10T18:51:04.797Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.525577528Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32632023-12-10T18:51:04.798Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.525618235Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32642023-12-10T18:51:04.798Z{"msg":"07c5c247-25ca-4706-bb90-1152c1d3873c WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.525675318Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32652023-12-10T18:51:04.798Z{"msg":"[0]R flush_numbers[0..12]: [43, 43, 43, 35, 34, 43, 39, 44, 43, 31, 44, 44]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.525716362Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263The guest has finished waiting for activation with:26
32662023-12-10T18:51:04.798ZWait for a query_work_queue command to finish before sending IO
32672023-12-10T18:51:04.798Z,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32682023-12-10T18:51:04.799Z{"msg":"[0]R generation[0..12]: [24, 24, 24, 18, 18, 24, 20, 25, 24, 16, 25, 25]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.52583436Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32692023-12-10T18:51:04.799Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.525912978Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32702023-12-10T18:51:04.799Z{"msg":"[1]R flush_numbers[0..12]: [43, 43, 43, 35, 34, 43, 39, 44, 43, 31, 44, 44]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.525941673Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32712023-12-10T18:51:04.799Z{"msg":"[1]R generation[0..12]: [24, 24, 24, 18, 18, 24, 20, 25, 24, 16, 25, 25]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.525963314Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32722023-12-10T18:51:04.799Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.525998203Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32732023-12-10T18:51:04.800Z{"msg":"[2]R flush_numbers[0..12]: [43, 43, 43, 35, 34, 43, 39, 44, 43, 31, 44, 44]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.526070786Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32742023-12-10T18:51:04.801Z{"msg":"[2]R generation[0..12]: [24, 24, 24, 18, 18, 24, 20, 25, 24, 16, 25, 25]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.526113194Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32752023-12-10T18:51:04.801Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.526184945Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32762023-12-10T18:51:04.801Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.526226273Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32772023-12-10T18:51:04.802Z{"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.526294346Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32782023-12-10T18:51:04.802Z{"msg":"Next flush: 45","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.526370391Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32792023-12-10T18:51:04.802Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.526413069Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32802023-12-10T18:51:04.802Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.526546335Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32812023-12-10T18:51:04.802Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.52658769Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32822023-12-10T18:51:04.802Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.5266441Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32832023-12-10T18:51:04.802Z{"msg":"07c5c247-25ca-4706-bb90-1152c1d3873c is now active with session: 608c8f54-17b4-4e5e-b0a4-8a056e96aec8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.526781419Z","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
32842023-12-10T18:51:04.803Zhostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32852023-12-10T18:51:04.803Z{"msg":"07c5c247-25ca-4706-bb90-1152c1d3873c Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.526869616Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32862023-12-10T18:51:04.803Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.526903389Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32872023-12-10T18:51:04.803Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.526957479Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32882023-12-10T18:51:04.803Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.526983073Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32892023-12-10T18:51:04.804Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.527053592Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32902023-12-10T18:51:04.804Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.527083324Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32912023-12-10T18:51:04.804Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.527137371Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32922023-12-10T18:51:04.804Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.527164856Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32932023-12-10T18:51:04.804Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.527219287Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32942023-12-10T18:51:04.804Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:03.527246756Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32952023-12-10T18:51:04.805Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30Loading write count information from file "/tmp/test_repair_verify.data"
32962023-12-10T18:51:04.805Z,"time":"2023-12-10T18:51:03.527300956Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32972023-12-10T18:51:04.806Z{"msg":"[1] Starts cmd_loop","v":0,"Run Repair workload
32982023-12-10T18:51:04.806Zname":"crucible","level":30,"time":"2023-12-10T18:51:03.527331978Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1263,"session_id":"608c8f54-17b4-4e5e-b0a4-8a056e96aec8"}
32992023-12-10T18:51:04.806Z01/30 Write block 83 len 24576 data: 4 3 4 5 4 4
33002023-12-10T18:51:04.806Z02/30 Read block 481 len 32768
33012023-12-10T18:51:04.806Z03/30 Write block 162 len 36864 data: 7 7 6 6 6 6 4 4 4
33022023-12-10T18:51:04.806Z04/30 Read block 95 len 16384
33032023-12-10T18:51:04.806Z05/30 Read block 586 len 4096
33042023-12-10T18:51:04.806Z06/30 Write block 433 len 12288 data: 3 3 3
33052023-12-10T18:51:04.807Z07/30 Read block 422 len 20480
33062023-12-10T18:51:04.807Z08/30 Write block 511 len 12288 data: 4 3 4
33072023-12-10T18:51:04.807Z09/30 Write block 364 len 36864 data: 3 3 3 4 4 4 3 2 3
33082023-12-10T18:51:04.807Z10/30 Flush
33092023-12-10T18:51:04.807Z11/30 Write block 407 len 24576 data: 2 2 2 2 2 2
33102023-12-10T18:51:04.808Z12/30 Write block 270 len 36864 data: 4 4 4 5 6 6 6 5 5
33112023-12-10T18:51:04.808Z13/30 Read block 230 len 24576
33122023-12-10T18:51:04.808Z14/30 Write block 0 len 12288 data: 2 2 3
33132023-12-10T18:51:04.808Z15/30 Write block 399 len 40960 data: 4 3 2 2 2 2 2 2 3 3
33142023-12-10T18:51:04.825Z16/30 Write block 117 len 32768 data: 5 6 6 6 6 6 6 6
33152023-12-10T18:51:04.825Z17/30 Write block 283 len 20480 data: 3 4 5 5 5
33162023-12-10T18:51:04.825Z18/30 Write block 214 len 12288 data: 6 6 7
33172023-12-10T18:51:04.825Z19/30 Read block 285 len 12288
33182023-12-10T18:51:04.825Z20/30 Flush
33192023-12-10T18:51:04.825Z21/30 Write block 5 len 36864 data: 2 2 2 2 2 2 2 2 2
33202023-12-10T18:51:04.826Z22/30 Read block 488 len 16384
33212023-12-10T18:51:04.826Z23/30 Read block 49 len 40960
33222023-12-10T18:51:04.826Z24/30 Write block 577 len 4096 data: 5
33232023-12-10T18:51:04.826Z25/30 Read block 392 len 20480
33242023-12-10T18:51:04.826Z26/30 Write block 217 len 32768 data: 7 6 4 4 5 5 4 3
33252023-12-10T18:51:04.826Z27/30 Read block 85 len 32768
33262023-12-10T18:51:04.826Z28/30 Write block 249 len 4096 data: 3
33272023-12-10T18:51:04.826Z29/30 Write block 375 len 20480 data: 3 3 3 4 4
33282023-12-10T18:51:04.826Z30/30 Read block 510 len 36864
33292023-12-10T18:51:04.827Z----------------------------------------------------------------
33302023-12-10T18:51:04.827Z Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:30
33312023-12-10T18:51:04.827ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
33322023-12-10T18:51:04.827Z 1 Acked 1000 Write 6 Done Sent Done false
33332023-12-10T18:51:04.827Z 2 Acked 1001 Read 8 Done Sent Done false
33342023-12-10T18:51:04.827Z 3 Acked 1002 Write 9 Done Sent Done false
33352023-12-10T18:51:04.827Z 4 Acked 1003 Read 4 Done Sent Done false
33362023-12-10T18:51:04.827Z 5 Acked 1004 Read 1 Done Sent Done false
33372023-12-10T18:51:04.827Z 6 Acked 1005 Write 3 Done Sent Done false
33382023-12-10T18:51:04.827Z 7 Acked 1006 Read 5 Done Sent Done false
33392023-12-10T18:51:04.827Z 8 Acked 1007 Write 3 Done Sent Done false
33402023-12-10T18:51:04.827Z 9 Acked 1008 Write 9 Done Sent Done false
33412023-12-10T18:51:04.827Z 10 Acked 1009 Flush 0 Done Sent Done false
33422023-12-10T18:51:04.828Z 11 Acked 1010 Write 6 Done Sent Done false
33432023-12-10T18:51:04.828Z 12 Acked 1011 Write 9 Done Sent Done false
33442023-12-10T18:51:04.828Z 13 Acked 1012 Read 6 Done Sent Done false
33452023-12-10T18:51:04.828Z 14 Acked 1013 Write 3 Done Sent Done false
33462023-12-10T18:51:04.828Z 15 Acked 1014 Write 10 Done Sent Done false
33472023-12-10T18:51:04.828Z 16 Acked 1015 Write 8 Done Sent Done false
33482023-12-10T18:51:04.828Z 17 Acked 1016 Write 5 Done Sent Done false
33492023-12-10T18:51:04.828Z 18 Acked 1017 Write 3 Done Sent Done false
33502023-12-10T18:51:04.828Z 19 Acked 1018 Read 3 Done Sent Done false
33512023-12-10T18:51:04.828Z 20 Acked 1019 Flush 0 Done Sent Done false
33522023-12-10T18:51:04.828Z 21 Acked 1020 Write 9 Done Sent Done false
33532023-12-10T18:51:04.828Z 22 Acked 1021 Read 4 Done Sent Done false
33542023-12-10T18:51:04.829Z 23 Acked 1022 Read 10 Done Sent Done false
33552023-12-10T18:51:04.829Z 24 Acked 1023 Write 1 Done Sent Done false
33562023-12-10T18:51:04.829Z 25 Acked 1024 Read 5 Done Sent Done false
33572023-12-10T18:51:04.829Z 26 Acked 1025 Write 8 Done Sent Done false
33582023-12-10T18:51:04.829Z 27 Acked 1026 Read 8 Done Sent Done false
33592023-12-10T18:51:04.829Z 28 Acked 1027 Write 1 Done Sent Done false
33602023-12-10T18:51:04.829Z 29 Acked 1028 Write 5 Done Sent Done false
33612023-12-10T18:51:04.829Z 30 Acked 1029 Read 9 Done Sent Done false
33622023-12-10T18:51:04.829Z STATES DS:0 DS:1 DS:2 TOTAL
33632023-12-10T18:51:04.829Z New 0 0 0 0
33642023-12-10T18:51:04.829Z Sent 0 30 0 30
33652023-12-10T18:51:04.829Z Done 30 0 30 60
33662023-12-10T18:51:04.830Z Skipped 0 0 0 0
33672023-12-10T18:51:04.830Z Error 0 0 0 0
33682023-12-10T18:51:04.830ZLast Flush: 1019 0 1019
33692023-12-10T18:51:04.830ZDownstairs last five completed:
33702023-12-10T18:51:04.830ZUpstairs last five completed: 30 29 28 27 26 25
33712023-12-10T18:51:04.830ZWrote out file "/tmp/test_repair_verify.data"
33722023-12-10T18:51:04.830Z
33732023-12-10T18:51:04.841ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
33742023-12-10T18:51:04.851ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
33752023-12-10T18:51:04.851Z 0 000-019 26 24 26 46 43 46 T F T <---
33762023-12-10T18:51:04.851Z 1 020-039 24 24 24 43 43 43 F F F
33772023-12-10T18:51:04.851Z 2 040-059 24 24 24 43 43 43 F F F
33782023-12-10T18:51:04.851Z 3 060-079 18 18 18 35 35 35 F F F
33792023-12-10T18:51:04.852Z 4 080-099 26 18 26 45 34 45 F F F <---
33802023-12-10T18:51:04.852Z 5 100-119 26 24 26 46 43 46 F F F <---
33812023-12-10T18:51:04.852Z 6 120-139 26 20 26 46 39 46 F F F <---
33822023-12-10T18:51:04.852Z 7 140-159 25 25 25 44 44 44 F F F
33832023-12-10T18:51:04.852Z 8 160-179 26 24 26 45 43 45 F F F <---
33842023-12-10T18:51:04.853Z 9 180-199 16 16 16 31 31 31 F F F
33852023-12-10T18:51:04.853Z 10 200-219 26 25 26 46 44 46 T F T <---
33862023-12-10T18:51:04.853Z 11 220-239 25 25 25 44 44 44 T F T <---
33872023-12-10T18:51:04.853Z 12 240-259 24 24 24 43 43 43 T F T <---
33882023-12-10T18:51:04.853Z 13 260-279 26 23 26 46 41 46 F F F <---
33892023-12-10T18:51:04.854Z 14 280-299 26 18 26 46 37 46 F F F <---
33902023-12-10T18:51:04.854Z 15 300-319 20 20 20 39 39 39 F F F
33912023-12-10T18:51:04.854Z 16 320-339 23 23 23 41 41 41 F F F
33922023-12-10T18:51:04.854Z 17 340-359 23 23 23 41 41 41 F F F
33932023-12-10T18:51:04.854Z 18 360-379 26 24 26 45 43 45 T F T <---
33942023-12-10T18:51:04.854Z 19 380-399 26 17 26 46 32 46 F F F <---
33952023-12-10T18:51:04.854Z 20 400-419 26 14 26 46 25 46 F F F <---
33962023-12-10T18:51:04.855Z 21 420-439 26 14 26 45 23 45 F F F <---
33972023-12-10T18:51:04.855Z 22 440-459 24 24 24 43 43 43 F F F
33982023-12-10T18:51:04.855Z 23 460-479 24 24 24 43 43 43 F F F
33992023-12-10T18:51:04.855Z 24 480-499 23 23 23 41 41 41 F F F
34002023-12-10T18:51:04.855Z 25 500-519 26 20 26 45 39 45 F F F <---
34012023-12-10T18:51:04.855Z 26 520-539 20 20 20 39 39 39 F F F
34022023-12-10T18:51:04.855Z 27 540-559 14 14 14 25 25 25 F F F
34032023-12-10T18:51:04.856Z 28 560-579 23 23 23 41 41 41 T F T <---
34042023-12-10T18:51:04.856Z 29 580-599 23 23 23 41 41 41 F F F
34052023-12-10T18:51:04.856ZMax gen: 26, Max flush: 46
34062023-12-10T18:51:04.856ZError: Difference in extent metadata found!
34072023-12-10T18:51:04.867ZA Difference in extent metadata is expected here
34082023-12-10T18:51:04.867ZOn loop 13
34092023-12-10T18:51:04.867Z
34102023-12-10T18:51:04.867ZVerifying data now
34112023-12-10T18:51:10.354ZNo change after verify
34122023-12-10T18:51:10.355ZLoop: 13 Downstairs dump after verify (and repair):
34132023-12-10T18:51:10.377ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
34142023-12-10T18:51:10.377Z 0 000-019 27 27 27 47 47 47 F F F
34152023-12-10T18:51:10.379Z 1 020-039 24 24 24 43 43 43 F F F
34162023-12-10T18:51:10.379Z 2 040-059 24 24 24 43 43 43 F F F
34172023-12-10T18:51:10.379Z 3 060-079 18 18 18 35 35 35 F F F
34182023-12-10T18:51:10.382Z 4 080-099 26 26 26 45 45 45 F F F
34192023-12-10T18:51:10.382Z 5 100-119 26 26 26 46 46 46 F F F
34202023-12-10T18:51:10.382Z 6 120-139 26 26 26 46 46 46 F F F
34212023-12-10T18:51:10.382Z 7 140-159 25 25 25 44 44 44 F F F
34222023-12-10T18:51:10.382Z 8 160-179 26 26 26 45 45 45 F F F
34232023-12-10T18:51:10.383Z 9 180-199 16 16 16 31 31 31 F F F
34242023-12-10T18:51:10.383Z 10 200-219 27 27 27 47 47 47 F F F
34252023-12-10T18:51:10.383Z 11 220-239 27 27 27 47 47 47 F F F
34262023-12-10T18:51:10.383Z 12 240-259 27 27 27 47 47 47 F F F
34272023-12-10T18:51:10.383Z 13 260-279 26 26 26 46 46 46 F F F
34282023-12-10T18:51:10.383Z 14 280-299 26 26 26 46 46 46 F F F
34292023-12-10T18:51:10.383Z 15 300-319 20 20 20 39 39 39 F F F
34302023-12-10T18:51:10.383Z 16 320-339 23 23 23 41 41 41 F F F
34312023-12-10T18:51:10.383Z 17 340-359 23 23 23 41 41 41 F F F
34322023-12-10T18:51:10.383Z 18 360-379 27 27 27 47 47 47 F F F
34332023-12-10T18:51:10.383Z 19 380-399 26 26 26 46 46 46 F F F
34342023-12-10T18:51:10.383Z 20 400-419 26 26 26 46 46 46 F F F
34352023-12-10T18:51:10.383Z 21 420-439 26 26 26 45 45 45 F F F
34362023-12-10T18:51:10.383Z 22 440-459 24 24 24 43 43 43 F F F
34372023-12-10T18:51:10.383Z 23 460-479 24 24 24 43 43 43 F F F
34382023-12-10T18:51:10.383Z 24 480-499 23 23 23 41 41 41 F F F
34392023-12-10T18:51:10.383Z 25 500-519 26 26 26 45 45 45 F F F
34402023-12-10T18:51:10.384Z 26 520-539 20 20 20 39 39 39 F F F
34412023-12-10T18:51:10.384Z 27 540-559 14 14 14 25 25 25 F F F
34422023-12-10T18:51:10.384Z 28 560-579 27 27 27 47 47 47 F F F
34432023-12-10T18:51:10.384Z 29 580-599 23 23 23 41 41 41 F F F
34442023-12-10T18:51:10.384ZMax gen: 27, Max flush: 47
34452023-12-10T18:51:10.384Z
34462023-12-10T18:51:10.384ZBegin loop 14
34472023-12-10T18:51:10.384ZDownstairs to restart: 2
34482023-12-10T18:51:10.412Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.411795984Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271}
34492023-12-10T18:51:10.413Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.412219862Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271}
34502023-12-10T18:51:10.413Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.412248993Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271}
34512023-12-10T18:51:10.418Z{"msg":"Crucible 1c313f8b-7e16-4b53-a52a-10de857cf71e has session id: 9e705f38-7e52-461c-9fe5-a029c86723c5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.412272261Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34522023-12-10T18:51:10.418Z{"msg":"Upstairs opts: Upstairs UUID: 1c313f8b-7e16-4b53-a52a-10de857cf71e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.412352801Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34532023-12-10T18:51:10.418Z{"msg":"Crucible stats registered with UUID: 1c313f8b-7e16-4b53-a52a-10de857cf71e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.412399309Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34542023-12-10T18:51:10.418ZCrucible runtime is spawned
34552023-12-10T18:51:10.418Z{"The guest has requested activation with gen:28
34562023-12-10T18:51:10.418Zmsg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.4126659Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5","task":"up_listen"}
34572023-12-10T18:51:10.418Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.412742752Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"looper":"0","session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34582023-12-10T18:51:10.419Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.41279542Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34592023-12-10T18:51:10.419Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.412858128Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34602023-12-10T18:51:10.420Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.413024113Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"looper":"2","session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34612023-12-10T18:51:10.420Z{"msg":"[0] 1c313f8b-7e16-4b53-a52a-10de857cf71e looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.4131879Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"looper":"0","session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34622023-12-10T18:51:10.420Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.41334307Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"looper":"1","session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34632023-12-10T18:51:10.420Z{"msg":"[2] 1c313f8b-7e16-4b53-a52a-10de857cf71e looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.413558481Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"looper":"2","session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34642023-12-10T18:51:10.420Z{"msg":"[1] 1c313f8b-7e16-4b53-a52a-10de857cf71e looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.413604551Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"looper":"1","session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34652023-12-10T18:51:10.420Z{"msg":"1c313f8b-7e16-4b53-a52a-10de857cf71e active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.413831998Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34662023-12-10T18:51:10.420Z{"msg":"Set desired generation to :28","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.41392988Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34672023-12-10T18:51:10.421Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.414075581Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34682023-12-10T18:51:10.421Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.414510175Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34692023-12-10T18:51:10.421Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.414587073Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34702023-12-10T18:51:10.421Z{"msg":"[0] 1c313f8b-7e16-4b53-a52a-10de857cf71e (9e705f38-7e52-461c-9fe5-a029c86723c5) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.414634354Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34712023-12-10T18:51:10.421Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.414657394Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34722023-12-10T18:51:10.421Z{"msg":"[0] client is_active_req TRUE, promote! session 9e705f38-7e52-461c-9fe5-a029c86723c5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.414679427Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34732023-12-10T18:51:10.421Z{"msg":"[1] 1c313f8b-7e16-4b53-a52a-10de857cf71e (9e705f38-7e52-461c-9fe5-a029c86723c5) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.414731031Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34742023-12-10T18:51:10.421Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.414771734Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34752023-12-10T18:51:10.421Z{"msg":"[1] client is_active_req TRUE, promote! session 9e705f38-7e52-461c-9fe5-a029c86723c5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.414813748Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34762023-12-10T18:51:10.421Z{"msg":"[2] 1c313f8b-7e16-4b53-a52a-10de857cf71e (9e705f38-7e52-461c-9fe5-a029c86723c5) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.415310946Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34772023-12-10T18:51:10.422Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.415397243Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34782023-12-10T18:51:10.422Z{"msg":"[2] client is_active_req TRUE, promote! session 9e705f38-7e52-461c-9fe5-a029c86723c5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.415482026Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34792023-12-10T18:51:10.422Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.415534091Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34802023-12-10T18:51:10.422Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.415575619Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34812023-12-10T18:51:10.423Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.41562645Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34822023-12-10T18:51:10.423Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.41567922Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34832023-12-10T18:51:10.423Z{"msg":"1c313f8b-7e16-4b53-a52a-10de857cf71e WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.415718341Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34842023-12-10T18:51:10.423Z{"msg":"1c313f8b-7e16-4b53-a52a-10de857cf71e WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.415797897Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34852023-12-10T18:51:10.448Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.416219236Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34862023-12-10T18:51:10.448Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.416315539Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34872023-12-10T18:51:10.448Z{"msg":"1c313f8b-7e16-4b53-a52a-10de857cf71e WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.416397654Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34882023-12-10T18:51:10.448Z{"msg":"[1] 1c313f8b-7e16-4b53-a52a-10de857cf71e (9e705f38-7e52-461c-9fe5-a029c86723c5) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.416499492Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34892023-12-10T18:51:10.448Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.416542273Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34902023-12-10T18:51:10.449Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:10.416582744Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34912023-12-10T18:51:10.449Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.416621622Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34922023-12-10T18:51:10.449Z{"msg":"[0] 1c313f8b-7e16-4b53-a52a-10de857cf71e (9e705f38-7e52-461c-9fe5-a029c86723c5) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.416663196Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34932023-12-10T18:51:10.450Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.416701976Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34942023-12-10T18:51:10.450Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:10.416740009Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34952023-12-10T18:51:10.450Z{"msg":"1c313f8b-7e16-4b53-a52a-10de857cf71e WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.416780533Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34962023-12-10T18:51:10.450Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.416957463Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34972023-12-10T18:51:10.450Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.4170342Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34982023-12-10T18:51:10.450Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.417059913Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
34992023-12-10T18:51:10.450Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.417152021Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35002023-12-10T18:51:10.450Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.417233415Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35012023-12-10T18:51:10.451Z{"msg":"[2] 1c313f8b-7e16-4b53-a52a-10de857cf71e (9e705f38-7e52-461c-9fe5-a029c86723c5) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.417297122Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35022023-12-10T18:51:10.451Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.417337493Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35032023-12-10T18:51:10.451Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:10.417390965Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35042023-12-10T18:51:10.451Z{"msg":"1c313f8b-7e16-4b53-a52a-10de857cf71e WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.417430341Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35052023-12-10T18:51:10.451Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.41749542Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35062023-12-10T18:51:10.451ZThe guest has finished waiting for activation with:28
35072023-12-10T18:51:10.452Z{"msg":"Wait for a query_work_queue command to finish before sending IO
35082023-12-10T18:51:10.452Z[0]R flush_numbers[0..12]: [47, 43, 43, 35, 45, 46, 46, 44, 45, 31, 47, 47]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.417539976Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35092023-12-10T18:51:10.452Z{"msg":"[0]R generation[0..12]: [27, 24, 24, 18, 26, 26, 26, 25, 26, 16, 27, 27]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.417599554Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35102023-12-10T18:51:10.452Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.417624731Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35112023-12-10T18:51:10.452Z{"msg":"[1]R flush_numbers[0..12]: [47, 43, 43, 35, 45, 46, 46, 44, 45, 31, 47, 47]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.417645716Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35122023-12-10T18:51:10.453Z{"msg":"[1]R generation[0..12]: [27, 24, 24, 18, 26, 26, 26, 25, 26, 16, 27, 27]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.417701299Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35132023-12-10T18:51:10.453Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.41772525Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35142023-12-10T18:51:10.454Z{"msg":"[2]R flush_numbers[0..12]: [47, 43, 43, 35, 45, 46, 46, 44, 45, 31, 47, 47]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.417746702Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35152023-12-10T18:51:10.454Z{"msg":"[2]R generation[0..12]: [27, 24, 24, 18, 26, 26, 26, 25, 26, 16, 27, 27]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.417767452Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35162023-12-10T18:51:10.454Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.417856457Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35172023-12-10T18:51:10.454Z{"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.41791642Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35182023-12-10T18:51:10.454Z{"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.417954063Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35192023-12-10T18:51:10.454Z{"msg":"Next flush: 48","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.418048186Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35202023-12-10T18:51:10.454Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.418076861Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35212023-12-10T18:51:10.455Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.418101892Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35222023-12-10T18:51:10.455Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.418156666Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35232023-12-10T18:51:10.455Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.418179691Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35242023-12-10T18:51:10.455Z{"msg":"1c313f8b-7e16-4b53-a52a-10de857cf71e is now active with session: 9e705f38-7e52-461c-9fe5-a029c86723c5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.418223786Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35252023-12-10T18:51:10.455Z{"msg":"1c313f8b-7e16-4b53-a52a-10de857cf71e Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.418246623Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35262023-12-10T18:51:10.455Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.418266803Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35272023-12-10T18:51:10.455Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.418330109Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35282023-12-10T18:51:10.455Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.418352407Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35292023-12-10T18:51:10.456Z{"msg":"1c313f8b-7e16-4b53-a52a-10de857cf71e Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.418372849Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35302023-12-10T18:51:10.456Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.418392086Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35312023-12-10T18:51:10.456Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.418411518Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35322023-12-10T18:51:10.456Z{"msg":"[1] All repairs completed, exit","v":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
35332023-12-10T18:51:10.457Z0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.418508098Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35342023-12-10T18:51:10.457Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.418587897Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35352023-12-10T18:51:10.457Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.418613692Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35362023-12-10T18:51:10.457Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.418634185Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35372023-12-10T18:51:10.458Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.418674478Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35382023-12-10T18:51:10.458Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.418696618Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35392023-12-10T18:51:10.458Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.418736918Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35402023-12-10T18:51:10.458Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:10.418758788Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1271,"session_id":"9e705f38-7e52-461c-9fe5-a029c86723c5"}
35412023-12-10T18:51:10.458ZLoading write count information from file "/tmp/test_repair_verify.data"
35422023-12-10T18:51:10.458ZRun Repair workload
35432023-12-10T18:51:10.459Z01/30 Write block 450 len 4096 data: 2
35442023-12-10T18:51:10.459Z02/30 Write block 234 len 16384 data: 4 4 4 4
35452023-12-10T18:51:10.459Z03/30 Write block 353 len 16384 data: 6 3 3 4
35462023-12-10T18:51:10.459Z04/30 Read block 199 len 36864
35472023-12-10T18:51:10.459Z05/30 Write block 462 len 40960 data: 3 3 3 3 3 4 4 4 3 4
35482023-12-10T18:51:10.459Z06/30 Read block 399 len 36864
35492023-12-10T18:51:10.474Z07/30 Read block 2 len 36864
35502023-12-10T18:51:10.474Z08/30 Write block 195 len 4096 data: 5
35512023-12-10T18:51:10.474Z09/30 Read block 327 len 36864
35522023-12-10T18:51:10.474Z10/30 Read block 129 len 24576
35532023-12-10T18:51:10.474Z11/30 Read block 51 len 16384
35542023-12-10T18:51:10.474Z12/30 Write block 198 len 32768 data: 4 4 3 3 3 2 2 3
35552023-12-10T18:51:10.475Z13/30 Read block 312 len 40960
35562023-12-10T18:51:10.475Z14/30 Write block 65 len 16384 data: 3 4 4 6
35572023-12-10T18:51:10.475Z15/30 Read block 275 len 12288
35582023-12-10T18:51:10.475Z16/30 Read block 16 len 28672
35592023-12-10T18:51:10.475Z17/30 Write block 469 len 12288 data: 5 4 5
35602023-12-10T18:51:10.475Z18/30 Write block 10 len 40960 data: 3 3 3 3 2 2 2 4 3 3
35612023-12-10T18:51:10.475Z19/30 Flush
35622023-12-10T18:51:10.475Z20/30 Write block 486 len 40960 data: 2 2 2 2 2 3 3 2 2 3
35632023-12-10T18:51:10.475Z21/30 Read block 403 len 40960
35642023-12-10T18:51:10.476Z22/30 Write block 516 len 40960 data: 4 4 3 4 4 3 3 3 5 5
35652023-12-10T18:51:10.894Z23/30 Write block 281 len 24576 data: 3 4 4 5 6 6
35662023-12-10T18:51:10.894Z24/30 Write block 440 len 16384 data: 5 6 3 3
35672023-12-10T18:51:10.894Z25/30 Read block 40 len 16384
35682023-12-10T18:51:10.894Z26/30 Flush
35692023-12-10T18:51:10.895Z27/30 Write block 248 len 24576 data: 3 4 3 3 2 2
35702023-12-10T18:51:10.895Z28/30 Read block 425 len 24576
35712023-12-10T18:51:10.895Z29/30 Write block 410 len 8192 data: 3 3
35722023-12-10T18:51:10.895Z30/30 Read block 537 len 12288
35732023-12-10T18:51:10.895Z----------------------------------------------------------------
35742023-12-10T18:51:10.895Z Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30
35752023-12-10T18:51:10.897ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
35762023-12-10T18:51:10.897Z 1 Acked 1000 Write 1 Done Done Done false
35772023-12-10T18:51:10.898Z 2 Acked 1001 Write 4 Done Done Sent false
35782023-12-10T18:51:10.898Z 3 Acked 1002 Write 4 Done Done Sent false
35792023-12-10T18:51:10.899Z 4 Acked 1003 Read 9 Done Done Sent false
35802023-12-10T18:51:10.899Z 5 Acked 1004 Write 10 Done Done Sent false
35812023-12-10T18:51:10.899Z 6 Acked 1005 Read 9 Done Done Sent false
35822023-12-10T18:51:10.899Z 7 Acked 1006 Read 9 Done Done Sent false
35832023-12-10T18:51:10.900Z 8 Acked 1007 Write 1 Done Done Sent false
35842023-12-10T18:51:10.900Z 9 Acked 1008 Read 9 Done Done Sent false
35852023-12-10T18:51:10.900Z 10 Acked 1009 Read 6 Done Done Sent false
35862023-12-10T18:51:10.900Z 11 Acked 1010 Read 4 Done Done Sent false
35872023-12-10T18:51:10.900Z 12 Acked 1011 Write 8 Done Done Sent false
35882023-12-10T18:51:10.900Z 13 Acked 1012 Read 10 Done Done Sent false
35892023-12-10T18:51:10.900Z 14 Acked 1013 Write 4 Done Done Sent false
35902023-12-10T18:51:10.901Z 15 Acked 1014 Read 3 Done Done Sent false
35912023-12-10T18:51:10.901Z 16 Acked 1015 Read 7 Done Done Sent false
35922023-12-10T18:51:10.901Z 17 Acked 1016 Write 3 Done Done Sent false
35932023-12-10T18:51:10.901Z 18 Acked 1017 Write 10 Done Done Sent false
35942023-12-10T18:51:10.901Z 19 Acked 1018 Flush 0 Done Done Sent false
35952023-12-10T18:51:10.902Z 20 Acked 1019 Write 10 Done Done Sent false
35962023-12-10T18:51:10.902Z 21 Acked 1020 Read 10 Done Done Sent false
35972023-12-10T18:51:10.902Z 22 Acked 1021 Write 10 Done Done Sent false
35982023-12-10T18:51:10.903Z 23 Acked 1022 Write 6 Done Done Sent false
35992023-12-10T18:51:10.903Z 24 Acked 1023 Write 4 Done Done Sent false
36002023-12-10T18:51:10.903Z 25 Acked 1024 Read 4 Done Done Sent false
36012023-12-10T18:51:10.903Z 26 Acked 1025 Flush 0 Done Done Sent false
36022023-12-10T18:51:10.903Z 27 Acked 1026 Write 6 Done Done Sent false
36032023-12-10T18:51:10.904Z 28 Acked 1027 Read 6 Done Done Sent false
36042023-12-10T18:51:10.904Z 29 Acked 1028 Write 2 Done Done Sent false
36052023-12-10T18:51:10.904Z 30 Acked 1029 Read 3 Done Sent Sent false
36062023-12-10T18:51:10.904Z STATES DS:0 DS:1 DS:2 TOTAL
36072023-12-10T18:51:10.904Z New 0 0 0 0
36082023-12-10T18:51:10.905Z Sent 0 1 29 30
36092023-12-10T18:51:10.905Z Done 30 29 1 60
36102023-12-10T18:51:10.905Z Skipped 0 0 0 0
36112023-12-10T18:51:10.905Z Error 0 0 0 0
36122023-12-10T18:51:10.905ZLast Flush: 1025 1025 0
36132023-12-10T18:51:10.909ZDownstairs last five completed:
36142023-12-10T18:51:10.909ZUpstairs last five completed: 30 29 28 27 26 25
36152023-12-10T18:51:10.910ZWrote out file "/tmp/test_repair_verify.data"
36162023-12-10T18:51:10.928Z
36172023-12-10T18:51:10.928ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
36182023-12-10T18:51:10.928ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36192023-12-10T18:51:10.929Z 0 000-019 28 28 27 48 48 47 F F F <---
36202023-12-10T18:51:10.929Z 1 020-039 24 24 24 43 43 43 F F F
36212023-12-10T18:51:10.929Z 2 040-059 24 24 24 43 43 43 F F F
36222023-12-10T18:51:10.930Z 3 060-079 28 28 18 48 48 35 F F F <---
36232023-12-10T18:51:10.930Z 4 080-099 26 26 26 45 45 45 F F F
36242023-12-10T18:51:10.930Z 5 100-119 26 26 26 46 46 46 F F F
36252023-12-10T18:51:10.930Z 6 120-139 26 26 26 46 46 46 F F F
36262023-12-10T18:51:10.931Z 7 140-159 25 25 25 44 44 44 F F F
36272023-12-10T18:51:10.931Z 8 160-179 26 26 26 45 45 45 F F F
36282023-12-10T18:51:10.931Z 9 180-199 28 28 16 48 48 31 F F F <---
36292023-12-10T18:51:10.931Z 10 200-219 28 28 27 48 48 47 F F F <---
36302023-12-10T18:51:10.931Z 11 220-239 28 28 27 48 48 47 F F F <---
36312023-12-10T18:51:10.932Z 12 240-259 27 27 27 47 47 47 T T F <---
36322023-12-10T18:51:10.932Z 13 260-279 26 26 26 46 46 46 F F F
36332023-12-10T18:51:10.932Z 14 280-299 28 28 26 49 49 46 F F F <---
36342023-12-10T18:51:10.932Z 15 300-319 20 20 20 39 39 39 F F F
36352023-12-10T18:51:10.932Z 16 320-339 23 23 23 41 41 41 F F F
36362023-12-10T18:51:10.932Z 17 340-359 28 28 23 48 48 41 F F F <---
36372023-12-10T18:51:10.933Z 18 360-379 27 27 27 47 47 47 F F F
36382023-12-10T18:51:10.933Z 19 380-399 26 26 26 46 46 46 F F F
36392023-12-10T18:51:10.933Z 20 400-419 26 26 26 46 46 46 T T F <---
36402023-12-10T18:51:10.933Z 21 420-439 26 26 26 45 45 45 F F F
36412023-12-10T18:51:10.933Z 22 440-459 28 28 24 49 49 43 F F T <---
36422023-12-10T18:51:10.934Z 23 460-479 28 28 24 48 48 43 F F F <---
36432023-12-10T18:51:10.934Z 24 480-499 28 28 23 49 49 41 F F F <---
36442023-12-10T18:51:10.934Z 25 500-519 28 28 26 49 49 45 F F F <---
36452023-12-10T18:51:10.935Z 26 520-539 28 28 20 49 49 39 F F F <---
36462023-12-10T18:51:10.935Z 27 540-559 14 14 14 25 25 25 F F F
36472023-12-10T18:51:10.935Z 28 560-579 27 27 27 47 47 47 F F F
36482023-12-10T18:51:10.935Z 29 580-599 23 23 23 41 41 41 F F F
36492023-12-10T18:51:10.936ZMax gen: 28, Max flush: 49
36502023-12-10T18:51:10.936ZError: Difference in extent metadata found!
36512023-12-10T18:51:10.936ZA Difference in extent metadata is expected here
36522023-12-10T18:51:10.937ZOn loop 14
36532023-12-10T18:51:10.937Z
36542023-12-10T18:51:10.937ZVerifying data now
36552023-12-10T18:51:15.857ZNo change after verify
36562023-12-10T18:51:15.857ZLoop: 14 Downstairs dump after verify (and repair):
36572023-12-10T18:51:15.876ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36582023-12-10T18:51:15.876Z 0 000-019 28 28 28 48 48 48 F F F
36592023-12-10T18:51:15.880Z 1 020-039 24 24 24 43 43 43 F F F
36602023-12-10T18:51:15.880Z 2 040-059 24 24 24 43 43 43 F F F
36612023-12-10T18:51:15.880Z 3 060-079 28 28 28 48 48 48 F F F
36622023-12-10T18:51:15.880Z 4 080-099 26 26 26 45 45 45 F F F
36632023-12-10T18:51:15.881Z 5 100-119 26 26 26 46 46 46 F F F
36642023-12-10T18:51:15.881Z 6 120-139 26 26 26 46 46 46 F F F
36652023-12-10T18:51:15.881Z 7 140-159 25 25 25 44 44 44 F F F
36662023-12-10T18:51:15.881Z 8 160-179 26 26 26 45 45 45 F F F
36672023-12-10T18:51:15.881Z 9 180-199 28 28 28 48 48 48 F F F
36682023-12-10T18:51:15.882Z 10 200-219 28 28 28 48 48 48 F F F
36692023-12-10T18:51:15.882Z 11 220-239 28 28 28 48 48 48 F F F
36702023-12-10T18:51:15.882Z 12 240-259 29 29 29 50 50 50 F F F
36712023-12-10T18:51:15.882Z 13 260-279 26 26 26 46 46 46 F F F
36722023-12-10T18:51:15.882Z 14 280-299 28 28 28 49 49 49 F F F
36732023-12-10T18:51:15.882Z 15 300-319 20 20 20 39 39 39 F F F
36742023-12-10T18:51:15.882Z 16 320-339 23 23 23 41 41 41 F F F
36752023-12-10T18:51:15.882Z 17 340-359 28 28 28 48 48 48 F F F
36762023-12-10T18:51:15.882Z 18 360-379 27 27 27 47 47 47 F F F
36772023-12-10T18:51:15.882Z 19 380-399 26 26 26 46 46 46 F F F
36782023-12-10T18:51:15.882Z 20 400-419 29 29 29 50 50 50 F F F
36792023-12-10T18:51:15.882Z 21 420-439 26 26 26 45 45 45 F F F
36802023-12-10T18:51:15.882Z 22 440-459 28 28 28 49 49 49 F F F
36812023-12-10T18:51:15.882Z 23 460-479 28 28 28 48 48 48 F F F
36822023-12-10T18:51:15.882Z 24 480-499 28 28 28 49 49 49 F F F
36832023-12-10T18:51:15.882Z 25 500-519 28 28 28 49 49 49 F F F
36842023-12-10T18:51:15.883Z 26 520-539 28 28 28 49 49 49 F F F
36852023-12-10T18:51:15.883Z 27 540-559 14 14 14 25 25 25 F F F
36862023-12-10T18:51:15.883Z 28 560-579 27 27 27 47 47 47 F F F
36872023-12-10T18:51:15.883Z 29 580-599 23 23 23 41 41 41 F F F
36882023-12-10T18:51:15.883ZMax gen: 29, Max flush: 50
36892023-12-10T18:51:15.886Z
36902023-12-10T18:51:15.886ZBegin loop 15
36912023-12-10T18:51:15.886ZDownstairs to restart: 0
36922023-12-10T18:51:15.911Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.909887941Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279}
36932023-12-10T18:51:15.911Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.910297935Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279}
36942023-12-10T18:51:15.916Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.910326069Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279}
36952023-12-10T18:51:15.916Z{"msg":"Crucible d95eb8dd-812a-4848-a252-95635f28eadd has session id: e24bf4f2-6332-4586-aa1d-ed59420e9952","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.910393065Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
36962023-12-10T18:51:15.916Z{"msg":"Upstairs opts: Upstairs UUID: d95eb8dd-812a-4848-a252-95635f28eadd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.9104526Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
36972023-12-10T18:51:15.916Z{Crucible runtime is spawned
36982023-12-10T18:51:15.916Z"msg":"Crucible stats registered with UUID: d95eb8dd-812a-4848-a252-95635f28eadd","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.910535393Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
36992023-12-10T18:51:15.916ZThe guest has requested activation with gen:30
37002023-12-10T18:51:15.916Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.910782143Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952","task":"up_listen"}
37012023-12-10T18:51:15.917Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.910864776Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37022023-12-10T18:51:15.917Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.910936933Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"looper":"0","session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37032023-12-10T18:51:15.917Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.911015631Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37042023-12-10T18:51:15.917Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.91123806Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"looper":"2","session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37052023-12-10T18:51:15.917Z{"msg":"[0] d95eb8dd-812a-4848-a252-95635f28eadd looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.911293682Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"looper":"0","session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37062023-12-10T18:51:15.917Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.911385103Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"looper":"1","session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37072023-12-10T18:51:15.917Z{"msg":"[2] d95eb8dd-812a-4848-a252-95635f28eadd looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.911465865Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"looper":"2","session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37082023-12-10T18:51:15.917Z{"msg":"[1] d95eb8dd-812a-4848-a252-95635f28eadd looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.911560109Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"looper":"1","session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37092023-12-10T18:51:15.917Z{"msg":"d95eb8dd-812a-4848-a252-95635f28eadd active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.912045372Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37102023-12-10T18:51:15.917Z{"msg":"Set desired generation to :30","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.9121212Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37112023-12-10T18:51:15.917Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.91214623Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37122023-12-10T18:51:15.919Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.91260243Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37132023-12-10T18:51:15.919Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.912721811Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37142023-12-10T18:51:15.919Z{"msg":"[1] d95eb8dd-812a-4848-a252-95635f28eadd (e24bf4f2-6332-4586-aa1d-ed59420e9952) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.913132147Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37152023-12-10T18:51:15.919Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.913188148Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37162023-12-10T18:51:15.919Z{"msg":"[2] d95eb8dd-812a-4848-a252-95635f28eadd (e24bf4f2-6332-4586-aa1d-ed59420e9952) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.913214489Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37172023-12-10T18:51:15.919Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.913235129Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37182023-12-10T18:51:15.919Z{"msg":"[1] client is_active_req TRUE, promote! session e24bf4f2-6332-4586-aa1d-ed59420e9952","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.913296629Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37192023-12-10T18:51:15.919Z{"msg":"[2] client is_active_req TRUE, promote! session e24bf4f2-6332-4586-aa1d-ed59420e9952","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.913363385Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37202023-12-10T18:51:15.926Z{"msg":"[0] d95eb8dd-812a-4848-a252-95635f28eadd (e24bf4f2-6332-4586-aa1d-ed59420e9952) New WaitActive WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.913939434Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37212023-12-10T18:51:15.926Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.914115134Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37222023-12-10T18:51:15.926Z{"msg":"[0] client is_active_req TRUE, promote! session e24bf4f2-6332-4586-aa1d-ed59420e9952","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.914162916Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37232023-12-10T18:51:15.926Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.914206528Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37242023-12-10T18:51:15.926Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.914241368Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37252023-12-10T18:51:15.926Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.914300436Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37262023-12-10T18:51:15.926Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.91433053Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37272023-12-10T18:51:15.926Z{"msg":"d95eb8dd-812a-4848-a252-95635f28eadd WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.914360537Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37282023-12-10T18:51:15.926Z{"msg":"d95eb8dd-812a-4848-a252-95635f28eadd WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.914421509Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37292023-12-10T18:51:15.926Z{"msg":"[1] d95eb8dd-812a-4848-a252-95635f28eadd (e24bf4f2-6332-4586-aa1d-ed59420e9952) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.914783319Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37302023-12-10T18:51:15.926Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.914840958Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37312023-12-10T18:51:15.927Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:15.914865141Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37322023-12-10T18:51:15.928Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.914885564Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37332023-12-10T18:51:15.928Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.915058891Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37342023-12-10T18:51:15.928Z{"msg":"d95eb8dd-812a-4848-a252-95635f28eadd WaitActive WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.915117078Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37352023-12-10T18:51:15.928Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.915141649Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37362023-12-10T18:51:15.928Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.9151616Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37372023-12-10T18:51:15.928Z{"msg":"[2] d95eb8dd-812a-4848-a252-95635f28eadd (e24bf4f2-6332-4586-aa1d-ed59420e9952) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.915350741Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37382023-12-10T18:51:15.928Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.915402164Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37392023-12-10T18:51:15.928Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.91542573Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37402023-12-10T18:51:15.928Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.915451279Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37412023-12-10T18:51:15.929Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:15.915472978Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37422023-12-10T18:51:15.929Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.915520844Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37432023-12-10T18:51:15.929Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.915560764Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37442023-12-10T18:51:15.929Z{"msg":"d95eb8dd-812a-4848-a252-95635f28eadd WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.915642231Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37452023-12-10T18:51:15.929Z{"msg":"d95eb8dd-812a-4848-a252-95635f28eadd WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.915668874Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37462023-12-10T18:51:15.929Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.915709887Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37472023-12-10T18:51:15.929Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.915731527Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37482023-12-10T18:51:15.929Z{"msg":"[0] d95eb8dd-812a-4848-a252-95635f28eadd (e24bf4f2-6332-4586-aa1d-ed59420e9952) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.91605172Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37492023-12-10T18:51:15.930Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.916115531Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37502023-12-10T18:51:15.930Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:15.91613957Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37512023-12-10T18:51:15.930Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.916159571Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37522023-12-10T18:51:15.930Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.916179731Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37532023-12-10T18:51:15.930Z{"msg":"d95eb8dd-812a-4848-a252-95635f28eadd WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.916228542Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37542023-12-10T18:51:15.930Z{"msg":"[0]R flush_numbers[0..12]: [48, 43, 43, 48, 45, 46, 46, 44, 45, 48, 48, 48]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.916320608Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37552023-12-10T18:51:15.930Z{"msg":"[0]R generation[0..12]: [28, 24, 24, 28, 26, 26, 26, 25, 26, 28, 28, 28]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.916371425Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37562023-12-10T18:51:15.931Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.916396577Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37572023-12-10T18:51:15.931Z{"msg":"[1]R flush_numbers[0..12]: [48, 43, 43, 48, 45, 46, 46, 44, 45, 48, 48, 48]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.916437883Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37582023-12-10T18:51:15.931Z{"msg":"[1]R generation[0..12]: [28, 24, 24, 28, 26, 26, 26, 25, 26, 28, 28, 28]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.916461547Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37592023-12-10T18:51:15.931Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.916518123Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37602023-12-10T18:51:15.931Z{"msg":"[2]R flush_numbers[0..12]: [48, 43, 43, 48, 45, 46, 46, 44, 45, 48, 48, 48]","v":0,"name":"The guest has finished waiting for activation with:30
37612023-12-10T18:51:15.931ZWait for a query_work_queue command to finish before sending IO
37622023-12-10T18:51:15.931Zcrucible","level":30,"time":"2023-12-10T18:51:15.916562398Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37632023-12-10T18:51:15.931Z{"msg":"[2]R generation[0..12]: [28, 24, 24, 28, 26, 26, 26, 25, 26, 28, 28, 28]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.916630762Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37642023-12-10T18:51:15.931Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.916657406Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37652023-12-10T18:51:15.931Z{"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.916690977Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37662023-12-10T18:51:15.931Z{"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.91672816Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37672023-12-10T18:51:15.931Z{"msg":"Next flush: 51","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.916787078Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37682023-12-10T18:51:15.931Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.916825854Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37692023-12-10T18:51:15.931Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.916880439Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37702023-12-10T18:51:15.931Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.916978324Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37712023-12-10T18:51:15.931Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.917035477Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37722023-12-10T18:51:15.931Z{"msg":"d95eb8dd-812a-4848-a252-95635f28eadd is now active with session: e24bf4f2-6332-4586-aa1d-ed59420e9952","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.917061264Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37732023-12-10T18:51:15.932Z{"msg":"d95eb8dd-812a-4848-a252-95635f28eadd Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.917116916Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37742023-12-10T18:51:15.932Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.917139541Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37752023-12-10T18:51:15.932Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.917159124Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37762023-12-10T18:51:15.932Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.917179444Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37772023-12-10T18:51:15.932Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.917219502Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37782023-12-10T18:51:15.932Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.917243089Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37792023-12-10T18:51:15.932Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.91727793Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37802023-12-10T18:51:15.932Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.917335548Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37812023-12-10T18:51:15.932Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.917433329Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37822023-12-10T18:51:15.933Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.917461768Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37832023-12-10T18:51:15.933Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.917482329Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37842023-12-10T18:51:15.933Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:15.917501638Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1279,"session_id":"e24bf4f2-6332-4586-aa1d-ed59420e9952"}
37852023-12-10T18:51:15.933ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
37862023-12-10T18:51:15.933ZLoading write count information from file "/tmp/test_repair_verify.data"
37872023-12-10T18:51:15.934ZRun Repair workload
37882023-12-10T18:51:15.934Z01/30 Write block 540 len 20480 data: 5 6 7 6 5
37892023-12-10T18:51:15.934Z02/30 Read block 476 len 40960
37902023-12-10T18:51:15.937Z03/30 Read block 358 len 4096
37912023-12-10T18:51:15.937Z04/30 Write block 582 len 8192 data: 3 3
37922023-12-10T18:51:15.937Z05/30 Read block 179 len 24576
37932023-12-10T18:51:15.938Z06/30 Write block 74 len 28672 data: 4 4 3 2 2 2 4
37942023-12-10T18:51:15.938Z07/30 Write block 296 len 36864 data: 3 3 3 3 2 2 2 3 4
37952023-12-10T18:51:15.938Z08/30 Read block 452 len 32768
37962023-12-10T18:51:15.938Z09/30 Write block 296 len 16384 data: 4 4 4 4
37972023-12-10T18:51:15.938Z10/30 Write block 77 len 40960 data: 3 3 3 5 4 4 5 4 5 6
37982023-12-10T18:51:15.938Z11/30 Write block 421 len 16384 data: 3 4 3 4
37992023-12-10T18:51:15.942Z12/30 Read block 98 len 4096
38002023-12-10T18:51:15.942Z13/30 Read block 519 len 28672
38012023-12-10T18:51:15.945Z14/30 Read block 206 len 16384
38022023-12-10T18:51:15.945Z15/30 Write block 119 len 32768 data: 7 7 7 7 7 7 5 5
38032023-12-10T18:51:15.945Z16/30 Read block 9 len 8192
38042023-12-10T18:51:15.948Z17/30 Read block 494 len 8192
38052023-12-10T18:51:15.949Z18/30 Flush
38062023-12-10T18:51:15.952Z19/30 Write block 561 len 8192 data: 2 2
38072023-12-10T18:51:15.952Z20/30 Flush
38082023-12-10T18:51:15.956Z21/30 Write block 503 len 24576 data: 4 4 4 5 4 4
38092023-12-10T18:51:15.956Z22/30 Read block 325 len 20480
38102023-12-10T18:51:15.959Z23/30 Write block 332 len 8192 data: 3 3
38112023-12-10T18:51:15.960Z24/30 Write block 486 len 28672 data: 3 3 3 3 3 4 4
38122023-12-10T18:51:15.965Z25/30 Read block 428 len 36864
38132023-12-10T18:51:15.965Z26/30 Read block 339 len 12288
38142023-12-10T18:51:15.989Z27/30 Write block 123 len 28672 data: 8 8 6 6 4 4 4
38152023-12-10T18:51:15.990Z28/30 Write block 224 len 36864 data: 4 3 3 3 3 3 3 3 3
38162023-12-10T18:51:15.990Z29/30 Read block 478 len 12288
38172023-12-10T18:51:15.990Z30/30 Write block 351 len 16384 data: 5 6 7 4
38182023-12-10T18:51:15.996Z----------------------------------------------------------------
38192023-12-10T18:51:15.996Z Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:30
38202023-12-10T18:51:15.997ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
38212023-12-10T18:51:15.997Z 1 Acked 1000 Write 5 Sent Done Done false
38222023-12-10T18:51:15.997Z 2 Acked 1001 Read 10 Sent Done Done false
38232023-12-10T18:51:15.997Z 3 Acked 1002 Read 1 Sent Done Done false
38242023-12-10T18:51:15.997Z 4 Acked 1003 Write 2 Sent Done Done false
38252023-12-10T18:51:15.997Z 5 Acked 1004 Read 6 Sent Done Done false
38262023-12-10T18:51:15.997Z 6 Acked 1005 Write 7 Sent Done Done false
38272023-12-10T18:51:15.998Z 7 Acked 1006 Write 9 Sent Done Done false
38282023-12-10T18:51:15.998Z 8 Acked 1007 Read 8 Sent Done Done false
38292023-12-10T18:51:15.999Z 9 Acked 1008 Write 4 Sent Done Done false
38302023-12-10T18:51:15.999Z 10 Acked 1009 Write 10 Sent Done Done false
38312023-12-10T18:51:15.999Z 11 Acked 1010 Write 4 Sent Done Done false
38322023-12-10T18:51:15.999Z 12 Acked 1011 Read 1 Sent Done Done false
38332023-12-10T18:51:15.999Z 13 Acked 1012 Read 7 Sent Done Done false
38342023-12-10T18:51:15.999Z 14 Acked 1013 Read 4 Sent Done Done false
38352023-12-10T18:51:15.999Z 15 Acked 1014 Write 8 Sent Done Done false
38362023-12-10T18:51:16.000Z 16 Acked 1015 Read 2 Sent Done Done false
38372023-12-10T18:51:16.000Z 17 Acked 1016 Read 2 Sent Done Done false
38382023-12-10T18:51:16.000Z 18 Acked 1017 Flush 0 Sent Done Done false
38392023-12-10T18:51:16.000Z 19 Acked 1018 Write 2 Sent Done Done false
38402023-12-10T18:51:16.000Z 20 Acked 1019 Flush 0 Sent Done Done false
38412023-12-10T18:51:16.000Z 21 Acked 1020 Write 6 Sent Done Done false
38422023-12-10T18:51:16.000Z 22 Acked 1021 Read 5 Sent Done Done false
38432023-12-10T18:51:16.001Z 23 Acked 1022 Write 2 Sent Done Done false
38442023-12-10T18:51:16.001Z 24 Acked 1023 Write 7 Sent Done Done false
38452023-12-10T18:51:16.001Z 25 Acked 1024 Read 9 Sent Done Done false
38462023-12-10T18:51:16.001Z 26 Acked 1025 Read 3 Sent Done Done false
38472023-12-10T18:51:16.001Z 27 Acked 1026 Write 7 Sent Done Done false
38482023-12-10T18:51:16.001Z 28 Acked 1027 Write 9 Sent Done Done false
38492023-12-10T18:51:16.001Z 29 Acked 1028 Read 3 Sent Done Done false
38502023-12-10T18:51:16.001Z 30 Acked 1029 Write 4 Sent Sent Sent false
38512023-12-10T18:51:16.001Z STATES DS:0 DS:1 DS:2 TOTAL
38522023-12-10T18:51:16.002Z New 0 0 0 0
38532023-12-10T18:51:16.002Z Sent 30 1 1 32
38542023-12-10T18:51:16.002Z Done 0 29 29 58
38552023-12-10T18:51:16.002Z Skipped 0 0 0 0
38562023-12-10T18:51:16.002Z Error 0 0 0 0
38572023-12-10T18:51:16.003ZLast Flush: 0 1019 1019
38582023-12-10T18:51:16.003ZDownstairs last five completed:
38592023-12-10T18:51:16.003ZUpstairs last five completed: 30 29 28 27 26 25
38602023-12-10T18:51:16.003ZWrote out file "/tmp/test_repair_verify.data"
38612023-12-10T18:51:16.003Z
38622023-12-10T18:51:16.003ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
38632023-12-10T18:51:16.008ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38642023-12-10T18:51:16.008Z 0 000-019 28 28 28 48 48 48 F F F
38652023-12-10T18:51:16.011Z 1 020-039 24 24 24 43 43 43 F F F
38662023-12-10T18:51:16.012Z 2 040-059 24 24 24 43 43 43 F F F
38672023-12-10T18:51:16.012Z 3 060-079 28 30 30 48 51 51 F F F <---
38682023-12-10T18:51:16.012Z 4 080-099 26 30 30 45 51 51 F F F <---
38692023-12-10T18:51:16.012Z 5 100-119 26 30 30 46 51 51 F F F <---
38702023-12-10T18:51:16.012Z 6 120-139 26 30 30 46 51 51 F T T <---
38712023-12-10T18:51:16.012Z 7 140-159 25 25 25 44 44 44 F F F
38722023-12-10T18:51:16.012Z 8 160-179 26 26 26 45 45 45 F F F
38732023-12-10T18:51:16.012Z 9 180-199 28 28 28 48 48 48 F F F
38742023-12-10T18:51:16.012Z 10 200-219 28 28 28 48 48 48 F F F
38752023-12-10T18:51:16.012Z 11 220-239 28 28 28 48 48 48 F T T <---
38762023-12-10T18:51:16.013Z 12 240-259 29 29 29 50 50 50 F F F
38772023-12-10T18:51:16.013Z 13 260-279 26 26 26 46 46 46 F F F
38782023-12-10T18:51:16.013Z 14 280-299 28 30 30 49 51 51 F F F <---
38792023-12-10T18:51:16.013Z 15 300-319 20 30 30 39 51 51 F F F <---
38802023-12-10T18:51:16.014Z 16 320-339 23 23 23 41 41 41 F T T <---
38812023-12-10T18:51:16.014Z 17 340-359 28 28 28 48 48 48 F T T <---
38822023-12-10T18:51:16.014Z 18 360-379 27 27 27 47 47 47 F F F
38832023-12-10T18:51:16.014Z 19 380-399 26 26 26 46 46 46 F F F
38842023-12-10T18:51:16.014Z 20 400-419 29 29 29 50 50 50 F F F
38852023-12-10T18:51:16.014Z 21 420-439 26 30 30 45 51 51 F F F <---
38862023-12-10T18:51:16.014Z 22 440-459 28 28 28 49 49 49 F F F
38872023-12-10T18:51:16.014Z 23 460-479 28 28 28 48 48 48 F F F
38882023-12-10T18:51:16.014Z 24 480-499 28 28 28 49 49 49 F T T <---
38892023-12-10T18:51:16.014Z 25 500-519 28 28 28 49 49 49 F T T <---
38902023-12-10T18:51:16.015Z 26 520-539 28 28 28 49 49 49 F F F
38912023-12-10T18:51:16.015Z 27 540-559 14 30 30 25 51 51 F F F <---
38922023-12-10T18:51:16.015Z 28 560-579 27 30 30 47 52 52 F F F <---
38932023-12-10T18:51:16.015Z 29 580-599 23 30 30 41 51 51 F F F <---
38942023-12-10T18:51:16.015ZMax gen: 30, Max flush: 52
38952023-12-10T18:51:16.015ZError: Difference in extent metadata found!
38962023-12-10T18:51:16.039ZA Difference in extent metadata is expected here
38972023-12-10T18:51:16.040ZOn loop 15
38982023-12-10T18:51:16.040Z
38992023-12-10T18:51:16.040ZVerifying data now
39002023-12-10T18:51:21.420ZNo change after verify
39012023-12-10T18:51:21.420ZLoop: 15 Downstairs dump after verify (and repair):
39022023-12-10T18:51:21.443ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
39032023-12-10T18:51:21.443Z 0 000-019 28 28 28 48 48 48 F F F
39042023-12-10T18:51:21.443Z 1 020-039 24 24 24 43 43 43 F F F
39052023-12-10T18:51:21.443Z 2 040-059 24 24 24 43 43 43 F F F
39062023-12-10T18:51:21.447Z 3 060-079 30 30 30 51 51 51 F F F
39072023-12-10T18:51:21.447Z 4 080-099 30 30 30 51 51 51 F F F
39082023-12-10T18:51:21.447Z 5 100-119 30 30 30 51 51 51 F F F
39092023-12-10T18:51:21.447Z 6 120-139 31 31 31 53 53 53 F F F
39102023-12-10T18:51:21.448Z 7 140-159 25 25 25 44 44 44 F F F
39112023-12-10T18:51:21.448Z 8 160-179 26 26 26 45 45 45 F F F
39122023-12-10T18:51:21.448Z 9 180-199 28 28 28 48 48 48 F F F
39132023-12-10T18:51:21.448Z 10 200-219 28 28 28 48 48 48 F F F
39142023-12-10T18:51:21.448Z 11 220-239 31 31 31 53 53 53 F F F
39152023-12-10T18:51:21.448Z 12 240-259 29 29 29 50 50 50 F F F
39162023-12-10T18:51:21.448Z 13 260-279 26 26 26 46 46 46 F F F
39172023-12-10T18:51:21.449Z 14 280-299 30 30 30 51 51 51 F F F
39182023-12-10T18:51:21.449Z 15 300-319 30 30 30 51 51 51 F F F
39192023-12-10T18:51:21.449Z 16 320-339 31 31 31 53 53 53 F F F
39202023-12-10T18:51:21.449Z 17 340-359 31 31 31 53 53 53 F F F
39212023-12-10T18:51:21.449Z 18 360-379 27 27 27 47 47 47 F F F
39222023-12-10T18:51:21.449Z 19 380-399 26 26 26 46 46 46 F F F
39232023-12-10T18:51:21.449Z 20 400-419 29 29 29 50 50 50 F F F
39242023-12-10T18:51:21.449Z 21 420-439 30 30 30 51 51 51 F F F
39252023-12-10T18:51:21.450Z 22 440-459 28 28 28 49 49 49 F F F
39262023-12-10T18:51:21.450Z 23 460-479 28 28 28 48 48 48 F F F
39272023-12-10T18:51:21.450Z 24 480-499 31 31 31 53 53 53 F F F
39282023-12-10T18:51:21.450Z 25 500-519 31 31 31 53 53 53 F F F
39292023-12-10T18:51:21.450Z 26 520-539 28 28 28 49 49 49 F F F
39302023-12-10T18:51:21.450Z 27 540-559 30 30 30 51 51 51 F F F
39312023-12-10T18:51:21.450Z 28 560-579 30 30 30 52 52 52 F F F
39322023-12-10T18:51:21.450Z 29 580-599 30 30 30 51 51 51 F F F
39332023-12-10T18:51:21.450ZMax gen: 31, Max flush: 53
39342023-12-10T18:51:21.456Z
39352023-12-10T18:51:21.456ZBegin loop 16
39362023-12-10T18:51:21.456ZDownstairs to restart: 0
39372023-12-10T18:51:21.887Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.47717653Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287}
39382023-12-10T18:51:21.887Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.477606234Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287}
39392023-12-10T18:51:21.897Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.477673312Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287}
39402023-12-10T18:51:21.897Z{"msg":"Crucible 3df50ab2-ab4a-4ef8-b352-419d250028e7 has session id: c5f4d6f9-fdde-4825-a50a-bfb535936b77","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.477806576Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39412023-12-10T18:51:21.897Z{"msg":"Upstairs opts: Upstairs UUID: 3df50ab2-ab4a-4ef8-b352-419d250028e7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.477860498Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39422023-12-10T18:51:21.898Z{"msg":"Crucible stats registered with UUID: 3df50ab2-ab4a-4ef8-b352-419d250028e7","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.477900218Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39432023-12-10T18:51:21.898ZCrucible runtime is spawned
39442023-12-10T18:51:21.898Z{"msg":"up_listen starts","v":0,"name":"crucible","level"The guest has requested activation with gen:32
39452023-12-10T18:51:21.898Z:30,"time":"2023-12-10T18:51:21.478166506Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77","task":"up_listen"}
39462023-12-10T18:51:21.898Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.478211714Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39472023-12-10T18:51:21.898Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.478258392Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39482023-12-10T18:51:21.898Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.478281882Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"looper":"0","session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39492023-12-10T18:51:21.898Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.478477949Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"looper":"2","session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39502023-12-10T18:51:21.899Z{"msg":"[0] 3df50ab2-ab4a-4ef8-b352-419d250028e7 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.478551281Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"looper":"0","session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39512023-12-10T18:51:21.899Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.478609152Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"looper":"1","session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39522023-12-10T18:51:21.899Z{"msg":"[2] 3df50ab2-ab4a-4ef8-b352-419d250028e7 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.47865673Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"looper":"2","session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39532023-12-10T18:51:21.899Z{"msg":"[1] 3df50ab2-ab4a-4ef8-b352-419d250028e7 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.478729048Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"looper":"1","session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39542023-12-10T18:51:21.899Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.479161968Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39552023-12-10T18:51:21.899Z{"msg":"3df50ab2-ab4a-4ef8-b352-419d250028e7 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.479237686Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39562023-12-10T18:51:21.899Z{"msg":"Set desired generation to :32","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.479312322Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39572023-12-10T18:51:21.899Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.479559595Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39582023-12-10T18:51:21.899Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.479641992Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39592023-12-10T18:51:21.900Z{"msg":"[1] 3df50ab2-ab4a-4ef8-b352-419d250028e7 (c5f4d6f9-fdde-4825-a50a-bfb535936b77) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.480126161Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39602023-12-10T18:51:21.900Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.48022316Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39612023-12-10T18:51:21.900Z{"msg":"[1] client is_active_req TRUE, promote! session c5f4d6f9-fdde-4825-a50a-bfb535936b77","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.48026874Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39622023-12-10T18:51:21.900Z{"msg":"[2] 3df50ab2-ab4a-4ef8-b352-419d250028e7 (c5f4d6f9-fdde-4825-a50a-bfb535936b77) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.480346016Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39632023-12-10T18:51:21.901Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.48038578Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39642023-12-10T18:51:21.901Z{"msg":"[2] client is_active_req TRUE, promote! session c5f4d6f9-fdde-4825-a50a-bfb535936b77","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.480412077Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39652023-12-10T18:51:21.901Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.481100877Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39662023-12-10T18:51:21.901Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.481175604Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39672023-12-10T18:51:21.902Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.481217974Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39682023-12-10T18:51:21.902Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.481316404Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39692023-12-10T18:51:21.904Z{"msg":"3df50ab2-ab4a-4ef8-b352-419d250028e7 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.481357744Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39702023-12-10T18:51:21.904Z{"msg":"3df50ab2-ab4a-4ef8-b352-419d250028e7 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.481381251Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39712023-12-10T18:51:21.904Z{"msg":"[1] 3df50ab2-ab4a-4ef8-b352-419d250028e7 (c5f4d6f9-fdde-4825-a50a-bfb535936b77) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.481898354Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39722023-12-10T18:51:21.904Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.481956185Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39732023-12-10T18:51:21.904Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:21.48198022Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39742023-12-10T18:51:21.904Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.482009913Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39752023-12-10T18:51:21.904Z{"msg":"[2] 3df50ab2-ab4a-4ef8-b352-419d250028e7 (c5f4d6f9-fdde-4825-a50a-bfb535936b77) New WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.482085651Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39762023-12-10T18:51:21.905Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.482127108Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39772023-12-10T18:51:21.905Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:21.482193768Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39782023-12-10T18:51:21.905Z{"msg":"3df50ab2-ab4a-4ef8-b352-419d250028e7 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.48223483Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39792023-12-10T18:51:21.905Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.48230087Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39802023-12-10T18:51:21.905Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.482345244Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39812023-12-10T18:51:21.905Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.482438423Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39822023-12-10T18:51:21.905Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.482524215Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39832023-12-10T18:51:21.905Z{"msg":"[0] 3df50ab2-ab4a-4ef8-b352-419d250028e7 (c5f4d6f9-fdde-4825-a50a-bfb535936b77) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.482571903Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39842023-12-10T18:51:21.905Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.482608634Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39852023-12-10T18:51:21.905Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.482674748Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39862023-12-10T18:51:21.905Z{"msg":"[0] client is_active_req TRUE, promote! session c5f4d6f9-fdde-4825-a50a-bfb535936b77","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.482751884Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39872023-12-10T18:51:21.905Z{"msg":"3df50ab2-ab4a-4ef8-b352-419d250028e7 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.482794259Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39882023-12-10T18:51:21.905Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.482862817Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39892023-12-10T18:51:21.905Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.48293147Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39902023-12-10T18:51:21.906Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.483357261Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39912023-12-10T18:51:21.906Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.483452437Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39922023-12-10T18:51:21.906Z{"msg":"3df50ab2-ab4a-4ef8-b352-419d250028e7 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.483515385Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39932023-12-10T18:51:21.906Z{"msg":"[0] 3df50ab2-ab4a-4ef8-b352-419d250028e7 (c5f4d6f9-fdde-4825-a50a-bfb535936b77) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.483982095Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39942023-12-10T18:51:21.906Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.48406495Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39952023-12-10T18:51:21.906Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:21.484139491Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39962023-12-10T18:51:21.906Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.484208658Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39972023-12-10T18:51:21.906Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.48427621Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39982023-12-10T18:51:21.906Z{"msg":"3df50ab2-ab4a-4ef8-b352-419d250028e7 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.484345792Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
39992023-12-10T18:51:21.907Z{"msg":"[0]R flush_numbers[0..12]: [48, 43, 43, 51, 51, 51, 53, 44, 45, 48, 48, 53]","v":0,"name":"crucible","level"The guest has finished waiting for activation with:32
40002023-12-10T18:51:21.908Z:30Wait for a query_work_queue command to finish before sending IO
40012023-12-10T18:51:21.908Z,"time":"2023-12-10T18:51:21.484418609Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40022023-12-10T18:51:21.908Z{"msg":"[0]R generation[0..12]: [28, 24, 24, 30, 30, 30, 31, 25, 26, 28, 28, 31]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.484586244Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40032023-12-10T18:51:21.908Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.484681185Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40042023-12-10T18:51:21.908Z{"msg":"[1]R flush_numbers[0..12]: [48, 43, 43, 51, 51, 51, 53, 44, 45, 48, 48, 53]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.48471598Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40052023-12-10T18:51:21.908Z{"msg":"[1]R generation[0..12]: [28, 24, 24, 30, 30, 30, 31, 25, 26, 28, 28, 31]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.48473828Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40062023-12-10T18:51:21.908Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.48478295Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40072023-12-10T18:51:21.908Z{"msg":"[2]R flush_numbers[0..12]: [48, 43, 43, 51, 51, 51, 53, 44, 45, 48, 48, 53]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.484806485Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40082023-12-10T18:51:21.908Z{"msg":"[2]R generation[0..12]: [28, 24, 24, 30, 30, 30, 31, 25, 26, 28, 28, 31]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.484861661Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40092023-12-10T18:51:21.909Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.48488532Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40102023-12-10T18:51:21.909Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.484909556Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40112023-12-10T18:51:21.910Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.484970818Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40122023-12-10T18:51:21.910Z{"msg":"Next flush: 54","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.484995239Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40132023-12-10T18:51:21.910Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.485015258Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40142023-12-10T18:51:21.910Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.485070507Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40152023-12-10T18:51:21.910Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.485142612Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40162023-12-10T18:51:21.910Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.485182023Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40172023-12-10T18:51:21.910Z{"msg":"3df50ab2-ab4a-4ef8-b352-419d250028e7 is now active with session: c5f4d6f9-fdde-4825-a50a-bfb535936b77","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.485249838Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40182023-12-10T18:51:21.910Z{"msg":"3df50ab2-ab4a-4ef8-b352-419d250028e7 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.485316973Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40192023-12-10T18:51:21.910Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.485385318Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40202023-12-10T18:51:21.910Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.485458434Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40212023-12-10T18:51:21.910Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.485527336Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40222023-12-10T18:51:21.911Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.485592364Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40232023-12-10T18:51:21.911Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.485616549Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40242023-12-10T18:51:21.911Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.485641543Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40252023-12-10T18:51:21.912Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.485676317Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40262023-12-10T18:51:21.912Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.485700356Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40272023-12-10T18:51:21.912Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.485795719Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40282023-12-10T18:51:21.912Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
40292023-12-10T18:51:21.912Z,"time":"2023-12-10T18:51:21.485818438Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40302023-12-10T18:51:21.912Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:21.485915634Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1287,"session_id":"c5f4d6f9-fdde-4825-a50a-bfb535936b77"}
40312023-12-10T18:51:21.912ZLoading write count information from file "/tmp/test_repair_verify.data"
40322023-12-10T18:51:21.912ZRun Repair workload
40332023-12-10T18:51:21.912Z01/30 Write block 249 len 32768 data: 5 4 4 3 3 2 2 2
40342023-12-10T18:51:21.912Z02/30 Read block 177 len 24576
40352023-12-10T18:51:21.912Z03/30 Write block 456 len 16384 data: 3 3 3 4
40362023-12-10T18:51:21.912Z04/30 Read block 547 len 24576
40372023-12-10T18:51:21.913Z05/30 Write block 29 len 4096 data: 2
40382023-12-10T18:51:21.913Z06/30 Write block 456 len 12288 data: 4 4 4
40392023-12-10T18:51:21.951Z07/30 Read block 352 len 24576
40402023-12-10T18:51:21.951Z08/30 Write block 465 len 20480 data: 4 4 5 5 6
40412023-12-10T18:51:21.951Z09/30 Read block 397 len 12288
40422023-12-10T18:51:21.951Z10/30 Read block 380 len 32768
40432023-12-10T18:51:21.952Z11/30 Write block 535 len 40960 data: 5 3 3 4 3 6 7 8 7 6
40442023-12-10T18:51:21.952Z12/30 Read block 161 len 40960
40452023-12-10T18:51:21.952Z13/30 Write block 321 len 40960 data: 2 3 2 2 2 2 3 3 3 3
40462023-12-10T18:51:21.952Z14/30 Read block 362 len 28672
40472023-12-10T18:51:21.952Z15/30 Write block 76 len 32768 data: 4 4 4 4 6 5 5 6
40482023-12-10T18:51:21.952Z16/30 Write block 241 len 8192 data: 7 8
40492023-12-10T18:51:21.953Z17/30 Write block 99 len 40960 data: 2 2 2 3 3 3 3 4 5 5
40502023-12-10T18:51:21.953Z18/30 Read block 542 len 16384
40512023-12-10T18:51:21.953Z19/30 Read block 423 len 8192
40522023-12-10T18:51:21.953Z20/30 Write block 291 len 16384 data: 3 3 3 3
40532023-12-10T18:51:21.953Z21/30 Write block 423 len 24576 data: 4 5 3 3 4 4
40542023-12-10T18:51:21.953Z22/30 Read block 138 len 32768
40552023-12-10T18:51:21.954Z23/30 Write block 266 len 20480 data: 2 2 3 3 5
40562023-12-10T18:51:21.954Z24/30 Read block 393 len 12288
40572023-12-10T18:51:21.954Z25/30 Write block 270 len 12288 data: 6 5 5
40582023-12-10T18:51:21.954Z26/30 Write block 559 len 24576 data: 2 2 3 3 2 2
40592023-12-10T18:51:21.954Z27/30 Write block 165 len 24576 data: 7 7 7 5 5 5
40602023-12-10T18:51:21.955Z28/30 Write block 541 len 32768 data: 8 9 8 7 5 6 4 4
40612023-12-10T18:51:21.955Z29/30 Read block 126 len 8192
40622023-12-10T18:51:21.956Z30/30 Read block 263 len 28672
40632023-12-10T18:51:21.956Z----------------------------------------------------------------
40642023-12-10T18:51:21.956Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30
40652023-12-10T18:51:21.956ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
40662023-12-10T18:51:21.957Z 1 Acked 1000 Write 8 Done Done Done false
40672023-12-10T18:51:21.957Z 2 Acked 1001 Read 6 Done Done Done false
40682023-12-10T18:51:21.957Z 3 Acked 1002 Write 4 Done Done Done false
40692023-12-10T18:51:21.957Z 4 Acked 1003 Read 6 Done Done Done false
40702023-12-10T18:51:21.957Z 5 Acked 1004 Write 1 Done Done Done false
40712023-12-10T18:51:21.958Z 6 Acked 1005 Write 3 Done Done Done false
40722023-12-10T18:51:21.958Z 7 Acked 1006 Read 6 Sent Done Done false
40732023-12-10T18:51:21.958Z 8 Acked 1007 Write 5 Sent Done Done false
40742023-12-10T18:51:21.958Z 9 Acked 1008 Read 3 Sent Done Done false
40752023-12-10T18:51:21.958Z 10 Acked 1009 Read 8 Sent Done Done false
40762023-12-10T18:51:21.959Z 11 Acked 1010 Write 10 Sent Done Done false
40772023-12-10T18:51:21.959Z 12 Acked 1011 Read 10 Sent Done Done false
40782023-12-10T18:51:21.959Z 13 Acked 1012 Write 10 Sent Done Done false
40792023-12-10T18:51:21.959Z 14 Acked 1013 Read 7 Sent Done Done false
40802023-12-10T18:51:21.959Z 15 Acked 1014 Write 8 Sent Done Done false
40812023-12-10T18:51:21.959Z 16 Acked 1015 Write 2 Sent Done Done false
40822023-12-10T18:51:21.959Z 17 Acked 1016 Write 10 Sent Done Done false
40832023-12-10T18:51:21.960Z 18 Acked 1017 Read 4 Sent Done Done false
40842023-12-10T18:51:21.960Z 19 Acked 1018 Read 2 Sent Done Done false
40852023-12-10T18:51:21.960Z 20 Acked 1019 Write 4 Sent Done Done false
40862023-12-10T18:51:21.960Z 21 Acked 1020 Write 6 Sent Done Done false
40872023-12-10T18:51:21.960Z 22 Acked 1021 Read 8 Sent Done Done false
40882023-12-10T18:51:21.960Z 23 Acked 1022 Write 5 Sent Done Done false
40892023-12-10T18:51:21.961Z 24 Acked 1023 Read 3 Sent Done Done false
40902023-12-10T18:51:21.961Z 25 Acked 1024 Write 3 Sent Done Done false
40912023-12-10T18:51:21.961Z 26 Acked 1025 Write 6 Sent Done Done false
40922023-12-10T18:51:21.961Z 27 Acked 1026 Write 6 Sent Done Done false
40932023-12-10T18:51:21.961Z 28 Acked 1027 Write 8 Sent Done Done false
40942023-12-10T18:51:21.962Z 29 Acked 1028 Read 2 Sent Done Done false
40952023-12-10T18:51:21.962Z 30 Acked 1029 Read 7 Sent Done Done false
40962023-12-10T18:51:21.962Z STATES DS:0 DS:1 DS:2 TOTAL
40972023-12-10T18:51:21.962Z New 0 0 0 0
40982023-12-10T18:51:21.962Z Sent 24 0 0 24
40992023-12-10T18:51:21.962Z Done 6 30 30 66
41002023-12-10T18:51:21.963Z Skipped 0 0 0 0
41012023-12-10T18:51:21.963Z Error 0 0 0 0
41022023-12-10T18:51:21.963ZLast Flush: 0 0 0
41032023-12-10T18:51:21.963ZDownstairs last five completed:
41042023-12-10T18:51:21.963ZUpstairs last five completed: 30 29 28 27 26 25
41052023-12-10T18:51:21.963ZWrote out file "/tmp/test_repair_verify.data"
41062023-12-10T18:51:21.963Z
41072023-12-10T18:51:21.964ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
41082023-12-10T18:51:21.964ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
41092023-12-10T18:51:21.965Z 0 000-019 28 28 28 48 48 48 F F F
41102023-12-10T18:51:21.966Z 1 020-039 24 24 24 43 43 43 T T T
41112023-12-10T18:51:21.966Z 2 040-059 24 24 24 43 43 43 F F F
41122023-12-10T18:51:21.966Z 3 060-079 30 30 30 51 51 51 F T T <---
41132023-12-10T18:51:21.966Z 4 080-099 30 30 30 51 51 51 F T T <---
41142023-12-10T18:51:21.966Z 5 100-119 30 30 30 51 51 51 F T T <---
41152023-12-10T18:51:21.966Z 6 120-139 31 31 31 53 53 53 F F F
41162023-12-10T18:51:21.967Z 7 140-159 25 25 25 44 44 44 F F F
41172023-12-10T18:51:21.967Z 8 160-179 26 26 26 45 45 45 F T T <---
41182023-12-10T18:51:21.967Z 9 180-199 28 28 28 48 48 48 F F F
41192023-12-10T18:51:21.967Z 10 200-219 28 28 28 48 48 48 F F F
41202023-12-10T18:51:21.967Z 11 220-239 31 31 31 53 53 53 F F F
41212023-12-10T18:51:21.967Z 12 240-259 29 29 29 50 50 50 T T T
41222023-12-10T18:51:21.968Z 13 260-279 26 26 26 46 46 46 F T T <---
41232023-12-10T18:51:21.968Z 14 280-299 30 30 30 51 51 51 F T T <---
41242023-12-10T18:51:21.968Z 15 300-319 30 30 30 51 51 51 F F F
41252023-12-10T18:51:21.968Z 16 320-339 31 31 31 53 53 53 F T T <---
41262023-12-10T18:51:21.968Z 17 340-359 31 31 31 53 53 53 F F F
41272023-12-10T18:51:21.968Z 18 360-379 27 27 27 47 47 47 F F F
41282023-12-10T18:51:21.969Z 19 380-399 26 26 26 46 46 46 F F F
41292023-12-10T18:51:21.969Z 20 400-419 29 29 29 50 50 50 F F F
41302023-12-10T18:51:21.969Z 21 420-439 30 30 30 51 51 51 F T T <---
41312023-12-10T18:51:21.969Z 22 440-459 28 28 28 49 49 49 T T T
41322023-12-10T18:51:21.969Z 23 460-479 28 28 28 48 48 48 F T T <---
41332023-12-10T18:51:21.969Z 24 480-499 31 31 31 53 53 53 F F F
41342023-12-10T18:51:21.970Z 25 500-519 31 31 31 53 53 53 F F F
41352023-12-10T18:51:21.970Z 26 520-539 28 28 28 49 49 49 F T T <---
41362023-12-10T18:51:21.970Z 27 540-559 30 30 30 51 51 51 F T T <---
41372023-12-10T18:51:21.970Z 28 560-579 30 30 30 52 52 52 F T T <---
41382023-12-10T18:51:21.970Z 29 580-599 30 30 30 51 51 51 F F F
41392023-12-10T18:51:21.976ZMax gen: 31, Max flush: 53
41402023-12-10T18:51:21.977ZError: Difference in extent metadata found!
41412023-12-10T18:51:21.977ZA Difference in extent metadata is expected here
41422023-12-10T18:51:21.977ZOn loop 16
41432023-12-10T18:51:21.977Z
41442023-12-10T18:51:21.977ZVerifying data now
41452023-12-10T18:51:28.075ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
41462023-12-10T18:51:28.080Z244d243
41472023-12-10T18:51:28.080Z< 6,
41482023-12-10T18:51:28.080Z245a245
41492023-12-10T18:51:28.080Z> 8,
41502023-12-10T18:51:28.081Zdiff found after verify
41512023-12-10T18:51:28.084ZLoop: 16 Downstairs dump after verify (and repair):
41522023-12-10T18:51:28.108ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
41532023-12-10T18:51:28.108Z 0 000-019 28 28 28 48 48 48 F F F
41542023-12-10T18:51:28.108Z 1 020-039 32 32 32 54 54 54 F F F
41552023-12-10T18:51:28.108Z 2 040-059 24 24 24 43 43 43 F F F
41562023-12-10T18:51:28.108Z 3 060-079 32 32 32 54 54 54 F F F
41572023-12-10T18:51:28.108Z 4 080-099 32 32 32 54 54 54 F F F
41582023-12-10T18:51:28.111Z 5 100-119 32 32 32 54 54 54 F F F
41592023-12-10T18:51:28.111Z 6 120-139 31 31 31 53 53 53 F F F
41602023-12-10T18:51:28.111Z 7 140-159 25 25 25 44 44 44 F F F
41612023-12-10T18:51:28.111Z 8 160-179 32 32 32 54 54 54 F F F
41622023-12-10T18:51:28.111Z 9 180-199 28 28 28 48 48 48 F F F
41632023-12-10T18:51:28.111Z 10 200-219 28 28 28 48 48 48 F F F
41642023-12-10T18:51:28.111Z 11 220-239 31 31 31 53 53 53 F F F
41652023-12-10T18:51:28.112Z 12 240-259 32 32 32 54 54 54 F F F
41662023-12-10T18:51:28.112Z 13 260-279 32 32 32 54 54 54 F F F
41672023-12-10T18:51:28.112Z 14 280-299 32 32 32 54 54 54 F F F
41682023-12-10T18:51:28.112Z 15 300-319 30 30 30 51 51 51 F F F
41692023-12-10T18:51:28.112Z 16 320-339 32 32 32 54 54 54 F F F
41702023-12-10T18:51:28.112Z 17 340-359 31 31 31 53 53 53 F F F
41712023-12-10T18:51:28.112Z 18 360-379 27 27 27 47 47 47 F F F
41722023-12-10T18:51:28.114Z 19 380-399 26 26 26 46 46 46 F F F
41732023-12-10T18:51:28.114Z 20 400-419 29 29 29 50 50 50 F F F
41742023-12-10T18:51:28.114Z 21 420-439 32 32 32 54 54 54 F F F
41752023-12-10T18:51:28.114Z 22 440-459 32 32 32 54 54 54 F F F
41762023-12-10T18:51:28.114Z 23 460-479 32 32 32 54 54 54 F F F
41772023-12-10T18:51:28.114Z 24 480-499 31 31 31 53 53 53 F F F
41782023-12-10T18:51:28.114Z 25 500-519 31 31 31 53 53 53 F F F
41792023-12-10T18:51:28.114Z 26 520-539 32 32 32 54 54 54 F F F
41802023-12-10T18:51:28.114Z 27 540-559 32 32 32 54 54 54 F F F
41812023-12-10T18:51:28.115Z 28 560-579 32 32 32 54 54 54 F F F
41822023-12-10T18:51:28.115Z 29 580-599 30 30 30 51 51 51 F F F
41832023-12-10T18:51:28.115ZMax gen: 32, Max flush: 54
41842023-12-10T18:51:28.118Z
41852023-12-10T18:51:28.118ZBegin loop 17
41862023-12-10T18:51:28.118ZDownstairs to restart: 2
41872023-12-10T18:51:28.144Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.143513445Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298}
41882023-12-10T18:51:28.148Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.143929248Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298}
41892023-12-10T18:51:28.148Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.144040607Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298}
41902023-12-10T18:51:28.148Z{"msg":"Crucible 939681d5-fc16-45fe-9f38-e8c6d3109369 has session id: 312017a2-8ca4-4666-8324-5de260a2c60e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.144066506Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
41912023-12-10T18:51:28.151Z{"msg":"Upstairs opts: Upstairs UUID: 939681d5-fc16-45fe-9f38-e8c6d3109369, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.144156323Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
41922023-12-10T18:51:28.151Z{"msg":"Crucible stats registered with UUID: 939681d5-fc16-45fe-9f38-e8c6d3109369","v":0,"name":"crucible","level":30Crucible runtime is spawned
41932023-12-10T18:51:28.152Z,"time":"2023-12-10T18:51:28.144234613Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
41942023-12-10T18:51:28.152Z{"msg":"up_listen starts","v":0,"name":"crucible"The guest has requested activation with gen:34
41952023-12-10T18:51:28.152Z,"level":30,"time":"2023-12-10T18:51:28.144461688Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e","task":"up_listen"}
41962023-12-10T18:51:28.152Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.14451351Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
41972023-12-10T18:51:28.152Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.144603938Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
41982023-12-10T18:51:28.152Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.14467816Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"looper":"1","session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
41992023-12-10T18:51:28.152Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.144742642Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"looper":"0","session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42002023-12-10T18:51:28.152Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.144888499Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"looper":"2","session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42012023-12-10T18:51:28.152Z{"msg":"[0] 939681d5-fc16-45fe-9f38-e8c6d3109369 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.144967084Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"looper":"0","session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42022023-12-10T18:51:28.153Z{"msg":"[1] 939681d5-fc16-45fe-9f38-e8c6d3109369 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.145043397Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"looper":"1","session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42032023-12-10T18:51:28.153Z{"msg":"[2] 939681d5-fc16-45fe-9f38-e8c6d3109369 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.145173368Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"looper":"2","session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42042023-12-10T18:51:28.153Z{"msg":"939681d5-fc16-45fe-9f38-e8c6d3109369 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.145507354Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42052023-12-10T18:51:28.153Z{"msg":"Set desired generation to :34","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.145585241Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42062023-12-10T18:51:28.153Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.145715392Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42072023-12-10T18:51:28.153Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.146022557Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42082023-12-10T18:51:28.154Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.146111704Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42092023-12-10T18:51:28.154Z{"msg":"[0] 939681d5-fc16-45fe-9f38-e8c6d3109369 (312017a2-8ca4-4666-8324-5de260a2c60e) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.146538287Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42102023-12-10T18:51:28.154Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.146618277Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42112023-12-10T18:51:28.154Z{"msg":"[0] client is_active_req TRUE, promote! session 312017a2-8ca4-4666-8324-5de260a2c60e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.14666286Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42122023-12-10T18:51:28.155Z{"msg":"[1] 939681d5-fc16-45fe-9f38-e8c6d3109369 (312017a2-8ca4-4666-8324-5de260a2c60e) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.146879287Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42132023-12-10T18:51:28.155Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.146945894Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42142023-12-10T18:51:28.155Z{"msg":"[1] client is_active_req TRUE, promote! session 312017a2-8ca4-4666-8324-5de260a2c60e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.146989104Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42152023-12-10T18:51:28.161Z{"msg":"[2] 939681d5-fc16-45fe-9f38-e8c6d3109369 (312017a2-8ca4-4666-8324-5de260a2c60e) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.147388377Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42162023-12-10T18:51:28.161Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.147457977Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42172023-12-10T18:51:28.161Z{"msg":"[2] client is_active_req TRUE, promote! session 312017a2-8ca4-4666-8324-5de260a2c60e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.147502119Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42182023-12-10T18:51:28.161Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.147539862Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42192023-12-10T18:51:28.161Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.147617673Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42202023-12-10T18:51:28.162Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.147702615Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42212023-12-10T18:51:28.163Z{"msg":"939681d5-fc16-45fe-9f38-e8c6d3109369 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.14774524Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42222023-12-10T18:51:28.163Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.14782173Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42232023-12-10T18:51:28.163Z{"msg":"939681d5-fc16-45fe-9f38-e8c6d3109369 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.147863776Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42242023-12-10T18:51:28.163Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.148345431Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42252023-12-10T18:51:28.163Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.148530633Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42262023-12-10T18:51:28.163Z{"msg":"[0] 939681d5-fc16-45fe-9f38-e8c6d3109369 (312017a2-8ca4-4666-8324-5de260a2c60e) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.148578645Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42272023-12-10T18:51:28.163Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.148615468Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42282023-12-10T18:51:28.163Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:28.148652901Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42292023-12-10T18:51:28.163Z{"msg":"939681d5-fc16-45fe-9f38-e8c6d3109369 WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.148700526Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42302023-12-10T18:51:28.164Z{"msg":"[1] 939681d5-fc16-45fe-9f38-e8c6d3109369 (312017a2-8ca4-4666-8324-5de260a2c60e) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.148726531Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42312023-12-10T18:51:28.164Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.148767115Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42322023-12-10T18:51:28.164Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:28.148789432Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42332023-12-10T18:51:28.164Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.148810228Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42342023-12-10T18:51:28.164Z{"msg":"939681d5-fc16-45fe-9f38-e8c6d3109369 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.14885889Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42352023-12-10T18:51:28.164Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.148881294Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42362023-12-10T18:51:28.164Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.148903289Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42372023-12-10T18:51:28.164Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.149057896Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42382023-12-10T18:51:28.164Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.149180982Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42392023-12-10T18:51:28.164Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.149224791Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42402023-12-10T18:51:28.164Z{"msg":"[2] 939681d5-fc16-45fe-9f38-e8c6d3109369 (312017a2-8ca4-4666-8324-5de260a2c60e) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.149300424Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42412023-12-10T18:51:28.168Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.149373037Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42422023-12-10T18:51:28.168Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:28.149443345Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42432023-12-10T18:51:28.168Z{"msg":"939681d5-fc16-45fe-9f38-e8c6d3109369 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.149510634Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42442023-12-10T18:51:28.168Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.149576839Z","hostname":"The guest has finished waiting for activation with:34
42452023-12-10T18:51:28.168Zip-10-150-1-104.us-west-2.compute.internal","pid":1298,"Wait for a query_work_queue command to finish before sending IO
42462023-12-10T18:51:28.168Zsession_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42472023-12-10T18:51:28.168Z{"msg":"[0]R flush_numbers[0..12]: [48, 54, 43, 54, 54, 54, 53, 44, 54, 48, 48, 53]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.14962546Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42482023-12-10T18:51:28.168Z{"msg":"[0]R generation[0..12]: [28, 32, 24, 32, 32, 32, 31, 25, 32, 28, 28, 31]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.149692876Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42492023-12-10T18:51:28.169Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.149736361Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42502023-12-10T18:51:28.169Z{"msg":"[1]R flush_numbers[0..12]: [48, 54, 43, 54, 54, 54, 53, 44, 54, 48, 48, 53]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.14978548Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42512023-12-10T18:51:28.169Z{"msg":"[1]R generation[0..12]: [28, 32, 24, 32, 32, 32, 31, 25, 32, 28, 28, 31]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.149876048Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42522023-12-10T18:51:28.169Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.149953212Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42532023-12-10T18:51:28.169Z{"msg":"[2]R flush_numbers[0..12]: [48, 54, 43, 54, 54, 54, 53, 44, 54, 48, 48, 53]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.150028477Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42542023-12-10T18:51:28.169Z{"msg":"[2]R generation[0..12]: [28, 32, 24, 32, 32, 32, 31, 25, 32, 28, 28, 31]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.150089091Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42552023-12-10T18:51:28.169Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.150115322Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42562023-12-10T18:51:28.169Z{"msg":"Max found gen is 33","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.150170544Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42572023-12-10T18:51:28.169Z{"msg":"Generation requested: 34 >= found:33","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.150192997Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42582023-12-10T18:51:28.169Z{"msg":"Next flush: 55","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.150212659Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42592023-12-10T18:51:28.169Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.150246396Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42602023-12-10T18:51:28.169Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.150275832Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42612023-12-10T18:51:28.169Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.150318892Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42622023-12-10T18:51:28.169Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.150341282Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42632023-12-10T18:51:28.169Z{"msg":"939681d5-fc16-45fe-9f38-e8c6d3109369 is now active with session: 312017a2-8ca4-4666-8324-5de260a2c60e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.150364323Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42642023-12-10T18:51:28.169Z{"msg":"939681d5-fc16-45fe-9f38-e8c6d3109369 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.150420161Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42652023-12-10T18:51:28.169Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.150453752Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42662023-12-10T18:51:28.170Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.150500695Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42672023-12-10T18:51:28.170Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.150523348Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42682023-12-10T18:51:28.170Z{"msg":"939681d5-fc16-45fe-9f38-e8c6d3109369 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.150543833Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42692023-12-10T18:51:28.170Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.150584579Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42702023-12-10T18:51:28.170Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.150606814Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42712023-12-10T18:51:28.170Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.150626538Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42722023-12-10T18:51:28.170Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
42732023-12-10T18:51:28.170Z,"time":"2023-12-10T18:51:28.150694074Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42742023-12-10T18:51:28.170Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.150786365Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42752023-12-10T18:51:28.170Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.150810938Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42762023-12-10T18:51:28.170Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.150831157Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42772023-12-10T18:51:28.170Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.150870534Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42782023-12-10T18:51:28.170Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.150892139Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42792023-12-10T18:51:28.170Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:28.150911735Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1298,"session_id":"312017a2-8ca4-4666-8324-5de260a2c60e"}
42802023-12-10T18:51:28.170ZLoading write count information from file "/tmp/test_repair_verify.data"
42812023-12-10T18:51:28.170ZRun Repair workload
42822023-12-10T18:51:28.170Z01/30 Write block 344 len 8192 data: 3 3
42832023-12-10T18:51:28.171Z02/30 Write block 591 len 16384 data: 4 3 3 3
42842023-12-10T18:51:28.171Z03/30 Write block 51 len 36864 data: 3 3 2 2 2 2 2 2 2
42852023-12-10T18:51:28.171Z04/30 Write block 284 len 16384 data: 6 7 7 6
42862023-12-10T18:51:28.171Z05/30 Write block 171 len 32768 data: 5 5 5 5 4 4 5 3
42872023-12-10T18:51:28.171Z06/30 Read block 294 len 24576
42882023-12-10T18:51:28.174Z07/30 Read block 522 len 40960
42892023-12-10T18:51:28.174Z08/30 Read block 530 len 4096
42902023-12-10T18:51:28.174Z09/30 Write block 81 len 16384 data: 6 6 7 5
42912023-12-10T18:51:28.175Z10/30 Write block 366 len 8192 data: 4 5
42922023-12-10T18:51:28.175Z11/30 Read block 576 len 12288
42932023-12-10T18:51:28.175Z12/30 Read block 181 len 4096
42942023-12-10T18:51:28.175Z13/30 Read block 28 len 16384
42952023-12-10T18:51:28.175Z14/30 Read block 207 len 12288
42962023-12-10T18:51:28.175Z15/30 Read block 338 len 8192
42972023-12-10T18:51:28.175Z16/30 Write block 281 len 8192 data: 4 5
42982023-12-10T18:51:28.178Z17/30 Read block 108 len 16384
42992023-12-10T18:51:28.178Z18/30 Read block 143 len 32768
43002023-12-10T18:51:28.181Z19/30 Read block 453 len 24576
43012023-12-10T18:51:28.184Z20/30 Write block 277 len 16384 data: 6 6 5 3
43022023-12-10T18:51:28.184Z21/30 Write block 79 len 36864 data: 5 7 7 7 8 6 6 7 5
43032023-12-10T18:51:28.184Z22/30 Read block 195 len 24576
43042023-12-10T18:51:28.190Z23/30 Write block 182 len 24576 data: 3 3 3 3 2 2
43052023-12-10T18:51:28.194Z24/30 Read block 474 len 28672
43062023-12-10T18:51:28.194Z25/30 Read block 218 len 16384
43072023-12-10T18:51:28.194Z26/30 Read block 369 len 12288
43082023-12-10T18:51:28.197Z27/30 Write block 417 len 20480 data: 2 3 3 3 4
43092023-12-10T18:51:28.197Z28/30 Write block 203 len 12288 data: 3 3 4
43102023-12-10T18:51:28.197Z29/30 Read block 146 len 28672
43112023-12-10T18:51:28.200Z30/30 Write block 360 len 24576 data: 5 3 3 3 4 4
43122023-12-10T18:51:28.200Z----------------------------------------------------------------
43132023-12-10T18:51:28.200Z Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30
43142023-12-10T18:51:28.200ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
43152023-12-10T18:51:28.200Z 1 Acked 1000 Write 2 Done Done Done false
43162023-12-10T18:51:28.200Z 2 Acked 1001 Write 4 Done Done Done false
43172023-12-10T18:51:28.201Z 3 Acked 1002 Write 9 Done Done Done false
43182023-12-10T18:51:28.201Z 4 Acked 1003 Write 4 Done Done Sent false
43192023-12-10T18:51:28.201Z 5 Acked 1004 Write 8 Done Done Sent false
43202023-12-10T18:51:28.201Z 6 Acked 1005 Read 6 Done Done Sent false
43212023-12-10T18:51:28.201Z 7 Acked 1006 Read 10 Done Done Sent false
43222023-12-10T18:51:28.204Z 8 Acked 1007 Read 1 Done Done Sent false
43232023-12-10T18:51:28.204Z 9 Acked 1008 Write 4 Done Done Sent false
43242023-12-10T18:51:28.205Z 10 Acked 1009 Write 2 Done Done Sent false
43252023-12-10T18:51:28.205Z 11 Acked 1010 Read 3 Done Done Sent false
43262023-12-10T18:51:28.205Z 12 Acked 1011 Read 1 Done Done Sent false
43272023-12-10T18:51:28.205Z 13 Acked 1012 Read 4 Done Done Sent false
43282023-12-10T18:51:28.205Z 14 Acked 1013 Read 3 Done Done Sent false
43292023-12-10T18:51:28.205Z 15 Acked 1014 Read 2 Done Done Sent false
43302023-12-10T18:51:28.205Z 16 Acked 1015 Write 2 Done Done Sent false
43312023-12-10T18:51:28.205Z 17 Acked 1016 Read 4 Done Done Sent false
43322023-12-10T18:51:28.205Z 18 Acked 1017 Read 8 Done Done Sent false
43332023-12-10T18:51:28.205Z 19 Acked 1018 Read 6 Done Done Sent false
43342023-12-10T18:51:28.205Z 20 Acked 1019 Write 4 Done Done Sent false
43352023-12-10T18:51:28.205Z 21 Acked 1020 Write 9 Done Done Sent false
43362023-12-10T18:51:28.205Z 22 Acked 1021 Read 6 Done Done Sent false
43372023-12-10T18:51:28.205Z 23 Acked 1022 Write 6 Done Done Sent false
43382023-12-10T18:51:28.205Z 24 Acked 1023 Read 7 Done Done Sent false
43392023-12-10T18:51:28.205Z 25 Acked 1024 Read 4 Done Done Sent false
43402023-12-10T18:51:28.205Z 26 Acked 1025 Read 3 Done Done Sent false
43412023-12-10T18:51:28.206Z 27 Acked 1026 Write 5 Done Done Sent false
43422023-12-10T18:51:28.206Z 28 Acked 1027 Write 3 Done Done Sent false
43432023-12-10T18:51:28.206Z 29 Acked 1028 Read 7 Done Done Sent false
43442023-12-10T18:51:28.206Z 30 Acked 1029 Write 6 Sent Sent Sent false
43452023-12-10T18:51:28.206Z STATES DS:0 DS:1 DS:2 TOTAL
43462023-12-10T18:51:28.206Z New 0 0 0 0
43472023-12-10T18:51:28.206Z Sent 1 1 27 29
43482023-12-10T18:51:28.206Z Done 29 29 3 61
43492023-12-10T18:51:28.206Z Skipped 0 0 0 0
43502023-12-10T18:51:28.206Z Error 0 0 0 0
43512023-12-10T18:51:28.206ZLast Flush: 0 0 0
43522023-12-10T18:51:28.206ZDownstairs last five completed:
43532023-12-10T18:51:28.206ZUpstairs last five completed: 30 29 28 27 26 25
43542023-12-10T18:51:28.206ZWrote out file "/tmp/test_repair_verify.data"
43552023-12-10T18:51:28.209Z
43562023-12-10T18:51:28.213ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
43572023-12-10T18:51:28.245ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
43582023-12-10T18:51:28.245Z 0 000-019 28 28 28 48 48 48 F F F
43592023-12-10T18:51:28.252Z 1 020-039 32 32 32 54 54 54 F F F
43602023-12-10T18:51:28.252Z 2 040-059 24 24 24 43 43 43 T T T
43612023-12-10T18:51:28.252Z 3 060-079 32 32 32 54 54 54 T T F <---
43622023-12-10T18:51:28.252Z 4 080-099 32 32 32 54 54 54 T T F <---
43632023-12-10T18:51:28.252Z 5 100-119 32 32 32 54 54 54 F F F
43642023-12-10T18:51:28.253Z 6 120-139 31 31 31 53 53 53 F F F
43652023-12-10T18:51:28.253Z 7 140-159 25 25 25 44 44 44 F F F
43662023-12-10T18:51:28.253Z 8 160-179 32 32 32 54 54 54 T T F <---
43672023-12-10T18:51:28.253Z 9 180-199 28 28 28 48 48 48 T T F <---
43682023-12-10T18:51:28.253Z 10 200-219 28 28 28 48 48 48 T T F <---
43692023-12-10T18:51:28.253Z 11 220-239 31 31 31 53 53 53 F F F
43702023-12-10T18:51:28.253Z 12 240-259 32 32 32 54 54 54 F F F
43712023-12-10T18:51:28.253Z 13 260-279 32 32 32 54 54 54 T T F <---
43722023-12-10T18:51:28.253Z 14 280-299 32 32 32 54 54 54 T T F <---
43732023-12-10T18:51:28.253Z 15 300-319 30 30 30 51 51 51 F F F
43742023-12-10T18:51:28.253Z 16 320-339 32 32 32 54 54 54 F F F
43752023-12-10T18:51:28.253Z 17 340-359 31 31 31 53 53 53 T T T
43762023-12-10T18:51:28.253Z 18 360-379 27 27 27 47 47 47 T T F <---
43772023-12-10T18:51:28.253Z 19 380-399 26 26 26 46 46 46 F F F
43782023-12-10T18:51:28.253Z 20 400-419 29 29 29 50 50 50 T T F <---
43792023-12-10T18:51:28.254Z 21 420-439 32 32 32 54 54 54 T T F <---
43802023-12-10T18:51:28.254Z 22 440-459 32 32 32 54 54 54 F F F
43812023-12-10T18:51:28.254Z 23 460-479 32 32 32 54 54 54 F F F
43822023-12-10T18:51:28.254Z 24 480-499 31 31 31 53 53 53 F F F
43832023-12-10T18:51:28.254Z 25 500-519 31 31 31 53 53 53 F F F
43842023-12-10T18:51:28.254Z 26 520-539 32 32 32 54 54 54 F F F
43852023-12-10T18:51:28.254Z 27 540-559 32 32 32 54 54 54 F F F
43862023-12-10T18:51:28.254Z 28 560-579 32 32 32 54 54 54 F F F
43872023-12-10T18:51:28.254Z 29 580-599 30 30 30 51 51 51 T T T
43882023-12-10T18:51:28.254ZMax gen: 32, Max flush: 54
43892023-12-10T18:51:28.254ZError: Difference in extent metadata found!
43902023-12-10T18:51:28.261ZA Difference in extent metadata is expected here
43912023-12-10T18:51:28.261ZOn loop 17
43922023-12-10T18:51:28.261Z
43932023-12-10T18:51:28.261ZVerifying data now
43942023-12-10T18:51:34.545ZNo change after verify
43952023-12-10T18:51:34.545ZLoop: 17 Downstairs dump after verify (and repair):
43962023-12-10T18:51:34.569ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
43972023-12-10T18:51:34.569Z 0 000-019 28 28 28 48 48 48 F F F
43982023-12-10T18:51:35.012Z 1 020-039 32 32 32 54 54 54 F F F
43992023-12-10T18:51:35.012Z 2 040-059 33 33 33 55 55 55 F F F
44002023-12-10T18:51:35.012Z 3 060-079 33 33 33 55 55 55 F F F
44012023-12-10T18:51:35.012Z 4 080-099 33 33 33 55 55 55 F F F
44022023-12-10T18:51:35.013Z 5 100-119 32 32 32 54 54 54 F F F
44032023-12-10T18:51:35.013Z 6 120-139 31 31 31 53 53 53 F F F
44042023-12-10T18:51:35.013Z 7 140-159 25 25 25 44 44 44 F F F
44052023-12-10T18:51:35.013Z 8 160-179 33 33 33 55 55 55 F F F
44062023-12-10T18:51:35.013Z 9 180-199 33 33 33 55 55 55 F F F
44072023-12-10T18:51:35.013Z 10 200-219 33 33 33 55 55 55 F F F
44082023-12-10T18:51:35.014Z 11 220-239 31 31 31 53 53 53 F F F
44092023-12-10T18:51:35.014Z 12 240-259 32 32 32 54 54 54 F F F
44102023-12-10T18:51:35.014Z 13 260-279 33 33 33 55 55 55 F F F
44112023-12-10T18:51:35.014Z 14 280-299 33 33 33 55 55 55 F F F
44122023-12-10T18:51:35.014Z 15 300-319 30 30 30 51 51 51 F F F
44132023-12-10T18:51:35.015Z 16 320-339 32 32 32 54 54 54 F F F
44142023-12-10T18:51:35.015Z 17 340-359 33 33 33 55 55 55 F F F
44152023-12-10T18:51:35.015Z 18 360-379 33 33 33 55 55 55 F F F
44162023-12-10T18:51:35.016Z 19 380-399 26 26 26 46 46 46 F F F
44172023-12-10T18:51:35.016Z 20 400-419 33 33 33 55 55 55 F F F
44182023-12-10T18:51:35.016Z 21 420-439 33 33 33 55 55 55 F F F
44192023-12-10T18:51:35.016Z 22 440-459 32 32 32 54 54 54 F F F
44202023-12-10T18:51:35.018Z 23 460-479 32 32 32 54 54 54 F F F
44212023-12-10T18:51:35.018Z 24 480-499 31 31 31 53 53 53 F F F
44222023-12-10T18:51:35.019Z 25 500-519 31 31 31 53 53 53 F F F
44232023-12-10T18:51:35.019Z 26 520-539 32 32 32 54 54 54 F F F
44242023-12-10T18:51:35.019Z 27 540-559 32 32 32 54 54 54 F F F
44252023-12-10T18:51:35.019Z 28 560-579 32 32 32 54 54 54 F F F
44262023-12-10T18:51:35.020Z 29 580-599 33 33 33 55 55 55 F F F
44272023-12-10T18:51:35.020ZMax gen: 33, Max flush: 55
44282023-12-10T18:51:35.020Z
44292023-12-10T18:51:35.020ZBegin loop 18
44302023-12-10T18:51:35.021ZDownstairs to restart: 2
44312023-12-10T18:51:35.021Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.60083439Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306}
44322023-12-10T18:51:35.021Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.60123375Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306}
44332023-12-10T18:51:35.021Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.601286445Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306}
44342023-12-10T18:51:35.022Z{"msg":"Crucible 7c027dcf-a439-4d74-aad1-4dfd593ade79 has session id: 54bf94ef-2ce0-4631-9f21-61a0d0caf0c8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.601311352Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44352023-12-10T18:51:35.022Z{"msg":"Upstairs opts: Upstairs UUID: 7c027dcf-a439-4d74-aad1-4dfd593ade79, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.601346792Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44362023-12-10T18:51:35.023Z{"msg":"Crucible stats registered with UUID: 7c027dcf-a439-4d74-aad1-4dfd593ade79","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.601396922Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44372023-12-10T18:51:35.023ZCrucible runtime is spawned
44382023-12-10T18:51:35.023ZThe guest has requested activation with gen:36
44392023-12-10T18:51:35.023Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.60172682Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"looper":"0","session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44402023-12-10T18:51:35.024Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.601797019Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8","task":"up_listen"}
44412023-12-10T18:51:35.024Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.601824834Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44422023-12-10T18:51:35.024Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.601845328Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44432023-12-10T18:51:35.024Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.602040166Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"looper":"1","session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44442023-12-10T18:51:35.024Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.602118064Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"looper":"2","session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44452023-12-10T18:51:35.024Z{"msg":"[0] 7c027dcf-a439-4d74-aad1-4dfd593ade79 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.602162102Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"looper":"0","session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44462023-12-10T18:51:35.024Z{"msg":"[1] 7c027dcf-a439-4d74-aad1-4dfd593ade79 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.602204356Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"looper":"1","session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44472023-12-10T18:51:35.025Z{"msg":"[2] 7c027dcf-a439-4d74-aad1-4dfd593ade79 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.60224664Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"looper":"2","session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44482023-12-10T18:51:35.025Z{"msg":"7c027dcf-a439-4d74-aad1-4dfd593ade79 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.602761074Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44492023-12-10T18:51:35.026Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.602816964Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44502023-12-10T18:51:35.026Z{"msg":"Set desired generation to :36","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.602841653Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44512023-12-10T18:51:35.026Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.603226213Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44522023-12-10T18:51:35.026Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.603411641Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44532023-12-10T18:51:35.026Z{"msg":"[1] 7c027dcf-a439-4d74-aad1-4dfd593ade79 (54bf94ef-2ce0-4631-9f21-61a0d0caf0c8) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.603656935Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44542023-12-10T18:51:35.026Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.603714474Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44552023-12-10T18:51:35.026Z{"msg":"[1] client is_active_req TRUE, promote! session 54bf94ef-2ce0-4631-9f21-61a0d0caf0c8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.60375737Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44562023-12-10T18:51:35.027Z{"msg":"[0] 7c027dcf-a439-4d74-aad1-4dfd593ade79 (54bf94ef-2ce0-4631-9f21-61a0d0caf0c8) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.603938154Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44572023-12-10T18:51:35.028Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.604111694Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44582023-12-10T18:51:35.028Z{"msg":"[0] client is_active_req TRUE, promote! session 54bf94ef-2ce0-4631-9f21-61a0d0caf0c8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.604160357Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44592023-12-10T18:51:35.028Z{"msg":"[2] 7c027dcf-a439-4d74-aad1-4dfd593ade79 (54bf94ef-2ce0-4631-9f21-61a0d0caf0c8) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.604449972Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44602023-12-10T18:51:35.028Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.604543539Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44612023-12-10T18:51:35.029Z{"msg":"[2] client is_active_req TRUE, promote! session 54bf94ef-2ce0-4631-9f21-61a0d0caf0c8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.604585717Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44622023-12-10T18:51:35.029Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.604760122Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44632023-12-10T18:51:35.029Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.604834415Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44642023-12-10T18:51:35.029Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.604929924Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44652023-12-10T18:51:35.029Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.605033657Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44662023-12-10T18:51:35.029Z{"msg":"7c027dcf-a439-4d74-aad1-4dfd593ade79 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.605077819Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44672023-12-10T18:51:35.029Z{"msg":"7c027dcf-a439-4d74-aad1-4dfd593ade79 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.60514402Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44682023-12-10T18:51:35.030Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.605453949Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44692023-12-10T18:51:35.030Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.605582642Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44702023-12-10T18:51:35.030Z{"msg":"7c027dcf-a439-4d74-aad1-4dfd593ade79 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.605645859Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44712023-12-10T18:51:35.030Z{"msg":"[0] 7c027dcf-a439-4d74-aad1-4dfd593ade79 (54bf94ef-2ce0-4631-9f21-61a0d0caf0c8) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.605689791Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44722023-12-10T18:51:35.030Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.605746952Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44732023-12-10T18:51:35.030Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:34.605786913Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44742023-12-10T18:51:35.030Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.605841143Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44752023-12-10T18:51:35.031Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.605880254Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44762023-12-10T18:51:35.031Z{"msg":"7c027dcf-a439-4d74-aad1-4dfd593ade79 WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.605967055Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44772023-12-10T18:51:35.031Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.606007183Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44782023-12-10T18:51:35.031Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.606046057Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44792023-12-10T18:51:35.031Z{"msg":"[1] 7c027dcf-a439-4d74-aad1-4dfd593ade79 (54bf94ef-2ce0-4631-9f21-61a0d0caf0c8) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.606115779Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44802023-12-10T18:51:35.031Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.606155159Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44812023-12-10T18:51:35.031Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:34.606211962Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44822023-12-10T18:51:35.031Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.606250993Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44832023-12-10T18:51:35.032Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.606305522Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44842023-12-10T18:51:35.032Z{"msg":"7c027dcf-a439-4d74-aad1-4dfd593ade79 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.606345712Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44852023-12-10T18:51:35.032Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.606406498Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"The guest has finished waiting for activation with:36
44862023-12-10T18:51:35.032Z}
44872023-12-10T18:51:35.032ZWait for a query_work_queue command to finish before sending IO
44882023-12-10T18:51:35.032Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.606450015Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44892023-12-10T18:51:35.032Z{"msg":"[2] 7c027dcf-a439-4d74-aad1-4dfd593ade79 (54bf94ef-2ce0-4631-9f21-61a0d0caf0c8) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.606519105Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44902023-12-10T18:51:35.032Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.606544616Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44912023-12-10T18:51:35.032Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:34.606564222Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44922023-12-10T18:51:35.032Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.606583961Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44932023-12-10T18:51:35.032Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.606610606Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44942023-12-10T18:51:35.032Z{"msg":"7c027dcf-a439-4d74-aad1-4dfd593ade79 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.606660677Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44952023-12-10T18:51:35.032Z{"msg":"[0]R flush_numbers[0..12]: [48, 54, 55, 55, 55, 54, 53, 44, 55, 55, 55, 53]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.606701462Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44962023-12-10T18:51:35.032Z{"msg":"[0]R generation[0..12]: [28, 32, 33, 33, 33, 32, 31, 25, 33, 33, 33, 31]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.606775967Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44972023-12-10T18:51:35.032Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.60685716Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44982023-12-10T18:51:35.061Z{"msg":"[1]R flush_numbers[0..12]: [48, 54, 55, 55, 55, 54, 53, 44, 55, 55, 55, 53]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.606888705Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
44992023-12-10T18:51:35.061Z{"msg":"[1]R generation[0..12]: [28, 32, 33, 33, 33, 32, 31, 25, 33, 33, 33, 31]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.606955036Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
45002023-12-10T18:51:35.062Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.606992575Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
45012023-12-10T18:51:35.062Z{"msg":"[2]R flush_numbers[0..12]: [48, 54, 55, 55, 55, 54, 53, 44, 55, 55, 55, 53]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.607032395Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
45022023-12-10T18:51:35.062Z{"msg":"[2]R generation[0..12]: [28, 32, 33, 33, 33, 32, 31, 25, 33, 33, 33, 31]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.607094076Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
45032023-12-10T18:51:35.062Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.607120896Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
45042023-12-10T18:51:35.064Z{"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.607165782Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
45052023-12-10T18:51:35.064Z{"msg":"Generation requested: 36 >= found:34","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.607188019Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
45062023-12-10T18:51:35.064Z{"msg":"Next flush: 56","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.607207407Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
45072023-12-10T18:51:35.064Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.607261251Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
45082023-12-10T18:51:35.065Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.607299729Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
45092023-12-10T18:51:35.065Z{"msg":"No initial repair work was required","v":0,"name":"crucible",Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
45102023-12-10T18:51:35.066Z"level":30,"time":"2023-12-10T18:51:34.607415929Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
45112023-12-10T18:51:35.066Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.607493105Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
45122023-12-10T18:51:35.066Z{"msg":"7c027dcf-a439-4d74-aad1-4dfd593ade79 is now active with session: 54bf94ef-2ce0-4631-9f21-61a0d0caf0c8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.60753311Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
45132023-12-10T18:51:35.066Z{"msg":"7c027dcf-a439-4d74-aad1-4dfd593ade79 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.607597469Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
45142023-12-10T18:51:35.066Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.607636023Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
45152023-12-10T18:51:35.066Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.607702455Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
45162023-12-10T18:51:35.067Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.607768022Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
45172023-12-10T18:51:35.067Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.607831924Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
45182023-12-10T18:51:35.067Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.607895984Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
45192023-12-10T18:51:35.069Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.60796186Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
45202023-12-10T18:51:35.070Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.608025523Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
45212023-12-10T18:51:35.070Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.608090097Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
45222023-12-10T18:51:35.070Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.608155799Z","Loading write count information from file "/tmp/test_repair_verify.data"
45232023-12-10T18:51:35.070Zhostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
45242023-12-10T18:51:35.070Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucibleRun Repair workload
45252023-12-10T18:51:35.071Z","level":30,"time":"2023-12-10T18:51:34.608317932Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
45262023-12-10T18:51:35.071Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:34.608388774Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1306,"session_id":"54bf94ef-2ce0-4631-9f21-61a0d0caf0c8"}
45272023-12-10T18:51:35.071Z01/30 Flush
45282023-12-10T18:51:35.071Z02/30 Write block 98 len 24576 data: 2 3 3 3 4 4
45292023-12-10T18:51:35.071Z03/30 Write block 3 len 8192 data: 2 2
45302023-12-10T18:51:35.071Z04/30 Read block 320 len 36864
45312023-12-10T18:51:35.072Z05/30 Write block 192 len 20480 data: 4 4 4 6 5
45322023-12-10T18:51:35.072Z06/30 Read block 81 len 32768
45332023-12-10T18:51:35.072Z07/30 Write block 530 len 24576 data: 5 5 5 4 5 6
45342023-12-10T18:51:35.072Z08/30 Read block 562 len 28672
45352023-12-10T18:51:35.072Z09/30 Read block 78 len 40960
45362023-12-10T18:51:35.075Z10/30 Flush
45372023-12-10T18:51:35.075Z11/30 Write block 547 len 40960 data: 5 5 2 2 2 2 2 2 3 3
45382023-12-10T18:51:35.075Z12/30 Read block 213 len 36864
45392023-12-10T18:51:35.075Z13/30 Write block 302 len 12288 data: 3 4 5
45402023-12-10T18:51:35.075Z14/30 Read block 140 len 16384
45412023-12-10T18:51:35.075Z15/30 Read block 439 len 32768
45422023-12-10T18:51:35.075Z16/30 Read block 89 len 16384
45432023-12-10T18:51:35.076Z17/30 Write block 483 len 36864 data: 2 2 2 4 4 4 4 4 5
45442023-12-10T18:51:35.076Z18/30 Read block 5 len 4096
45452023-12-10T18:51:35.076Z19/30 Flush
45462023-12-10T18:51:35.076Z20/30 Write block 341 len 28672 data: 2 2 3 4 4 3 5
45472023-12-10T18:51:35.076Z21/30 Read block 116 len 36864
45482023-12-10T18:51:35.077Z22/30 Write block 314 len 4096 data: 2
45492023-12-10T18:51:35.077Z23/30 Read block 445 len 16384
45502023-12-10T18:51:35.077Z24/30 Write block 445 len 20480 data: 3 3 3 2 2
45512023-12-10T18:51:35.077Z25/30 Write block 278 len 12288 data: 7 6 4
45522023-12-10T18:51:35.077Z26/30 Read block 23 len 36864
45532023-12-10T18:51:35.077Z27/30 Read block 198 len 24576
45542023-12-10T18:51:35.077Z28/30 Write block 154 len 16384 data: 2 2 2 6
45552023-12-10T18:51:35.078Z29/30 Write block 163 len 20480 data: 8 7 8 8 8
45562023-12-10T18:51:35.078Z30/30 Write block 43 len 36864 data: 3 3 4 4 4 4 4 4 4
45572023-12-10T18:51:35.078Z----------------------------------------------------------------
45582023-12-10T18:51:35.078Z Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30
45592023-12-10T18:51:35.078ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
45602023-12-10T18:51:35.078Z 1 Acked 1000 Flush 0 Done Done Sent false
45612023-12-10T18:51:35.079Z 2 Acked 1001 Write 6 Done Done Sent false
45622023-12-10T18:51:35.079Z 3 Acked 1002 Write 2 Done Done Sent false
45632023-12-10T18:51:35.079Z 4 Acked 1003 Read 9 Done Done Sent false
45642023-12-10T18:51:35.079Z 5 Acked 1004 Write 5 Done Done Sent false
45652023-12-10T18:51:35.079Z 6 Acked 1005 Read 8 Done Done Sent false
45662023-12-10T18:51:35.079Z 7 Acked 1006 Write 6 Done Done Sent false
45672023-12-10T18:51:35.079Z 8 Acked 1007 Read 7 Done Done Sent false
45682023-12-10T18:51:35.080Z 9 Acked 1008 Read 10 Done Done Sent false
45692023-12-10T18:51:35.080Z 10 Acked 1009 Flush 0 Done Done Sent false
45702023-12-10T18:51:35.080Z 11 Acked 1010 Write 10 Done Done Sent false
45712023-12-10T18:51:35.080Z 12 Acked 1011 Read 9 Done Done Sent false
45722023-12-10T18:51:35.080Z 13 Acked 1012 Write 3 Done Done Sent false
45732023-12-10T18:51:35.080Z 14 Acked 1013 Read 4 Done Done Sent false
45742023-12-10T18:51:35.081Z 15 Acked 1014 Read 8 Done Done Sent false
45752023-12-10T18:51:35.081Z 16 Acked 1015 Read 4 Done Done Sent false
45762023-12-10T18:51:35.081Z 17 Acked 1016 Write 9 Done Done Sent false
45772023-12-10T18:51:35.081Z 18 Acked 1017 Read 1 Done Done Sent false
45782023-12-10T18:51:35.081Z 19 Acked 1018 Flush 0 Done Done Sent false
45792023-12-10T18:51:35.081Z 20 Acked 1019 Write 7 Done Done Sent false
45802023-12-10T18:51:35.082Z 21 Acked 1020 Read 9 Done Done Sent false
45812023-12-10T18:51:35.082Z 22 Acked 1021 Write 1 Done Done Sent false
45822023-12-10T18:51:35.082Z 23 Acked 1022 Read 4 Done Done Sent false
45832023-12-10T18:51:35.082Z 24 Acked 1023 Write 5 Done Done Sent false
45842023-12-10T18:51:35.082Z 25 Acked 1024 Write 3 Done Done Sent false
45852023-12-10T18:51:35.082Z 26 Acked 1025 Read 9 Done Done Sent false
45862023-12-10T18:51:35.082Z 27 Acked 1026 Read 6 Done Done Sent false
45872023-12-10T18:51:35.083Z 28 Acked 1027 Write 4 Done Done Sent false
45882023-12-10T18:51:35.083Z 29 Acked 1028 Write 5 Done Done Sent false
45892023-12-10T18:51:35.083Z 30 Acked 1029 Write 9 Sent Sent Sent false
45902023-12-10T18:51:35.083Z STATES DS:0 DS:1 DS:2 TOTAL
45912023-12-10T18:51:35.083Z New 0 0 0 0
45922023-12-10T18:51:35.083Z Sent 1 1 30 32
45932023-12-10T18:51:35.084Z Done 29 29 0 58
45942023-12-10T18:51:35.084Z Skipped 0 0 0 0
45952023-12-10T18:51:35.084Z Error 0 0 0 0
45962023-12-10T18:51:35.084ZLast Flush: 1018 1018 0
45972023-12-10T18:51:35.084ZDownstairs last five completed:
45982023-12-10T18:51:35.105ZUpstairs last five completed: 30 29 28 27 26 25
45992023-12-10T18:51:35.105ZWrote out file "/tmp/test_repair_verify.data"
46002023-12-10T18:51:35.106Z
46012023-12-10T18:51:35.106ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
46022023-12-10T18:51:35.106ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
46032023-12-10T18:51:35.106Z 0 000-019 36 36 28 57 57 48 F F F <---
46042023-12-10T18:51:35.106Z 1 020-039 32 32 32 54 54 54 F F F
46052023-12-10T18:51:35.107Z 2 040-059 33 33 33 55 55 55 T T F <---
46062023-12-10T18:51:35.107Z 3 060-079 33 33 33 55 55 55 F F F
46072023-12-10T18:51:35.109Z 4 080-099 36 36 33 57 57 55 F F F <---
46082023-12-10T18:51:35.109Z 5 100-119 36 36 32 57 57 54 F F F <---
46092023-12-10T18:51:35.109Z 6 120-139 31 31 31 53 53 53 F F F
46102023-12-10T18:51:35.109Z 7 140-159 25 25 25 44 44 44 T T F <---
46112023-12-10T18:51:35.109Z 8 160-179 33 33 33 55 55 55 T T F <---
46122023-12-10T18:51:35.109Z 9 180-199 36 36 33 57 57 55 F F F <---
46132023-12-10T18:51:35.110Z 10 200-219 33 33 33 55 55 55 F F F
46142023-12-10T18:51:35.110Z 11 220-239 31 31 31 53 53 53 F F F
46152023-12-10T18:51:35.110Z 12 240-259 32 32 32 54 54 54 F F F
46162023-12-10T18:51:35.110Z 13 260-279 33 33 33 55 55 55 T T F <---
46172023-12-10T18:51:35.111Z 14 280-299 33 33 33 55 55 55 T T F <---
46182023-12-10T18:51:35.111Z 15 300-319 36 36 30 58 58 51 T T F <---
46192023-12-10T18:51:35.111Z 16 320-339 32 32 32 54 54 54 F F F
46202023-12-10T18:51:35.111Z 17 340-359 33 33 33 55 55 55 T T F <---
46212023-12-10T18:51:35.112Z 18 360-379 33 33 33 55 55 55 F F F
46222023-12-10T18:51:35.112Z 19 380-399 26 26 26 46 46 46 F F F
46232023-12-10T18:51:35.112Z 20 400-419 33 33 33 55 55 55 F F F
46242023-12-10T18:51:35.112Z 21 420-439 33 33 33 55 55 55 F F F
46252023-12-10T18:51:35.112Z 22 440-459 32 32 32 54 54 54 T T F <---
46262023-12-10T18:51:35.113Z 23 460-479 32 32 32 54 54 54 F F F
46272023-12-10T18:51:35.113Z 24 480-499 36 36 31 58 58 53 F F F <---
46282023-12-10T18:51:35.113Z 25 500-519 31 31 31 53 53 53 F F F
46292023-12-10T18:51:35.114Z 26 520-539 36 36 32 57 57 54 F F F <---
46302023-12-10T18:51:35.114Z 27 540-559 36 36 32 58 58 54 F F F <---
46312023-12-10T18:51:35.115Z 28 560-579 32 32 32 54 54 54 F F F
46322023-12-10T18:51:35.115Z 29 580-599 33 33 33 55 55 55 F F F
46332023-12-10T18:51:35.115ZMax gen: 36, Max flush: 58
46342023-12-10T18:51:35.116ZError: Difference in extent metadata found!
46352023-12-10T18:51:35.116ZA Difference in extent metadata is expected here
46362023-12-10T18:51:35.116ZOn loop 18
46372023-12-10T18:51:35.117Z
46382023-12-10T18:51:35.117ZVerifying data now
46392023-12-10T18:51:40.149ZNo change after verify
46402023-12-10T18:51:40.149ZLoop: 18 Downstairs dump after verify (and repair):
46412023-12-10T18:51:40.173ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
46422023-12-10T18:51:40.173Z 0 000-019 36 36 36 57 57 57 F F F
46432023-12-10T18:51:40.173Z 1 020-039 32 32 32 54 54 54 F F F
46442023-12-10T18:51:40.194Z 2 040-059 37 37 37 59 59 59 F F F
46452023-12-10T18:51:40.194Z 3 060-079 33 33 33 55 55 55 F F F
46462023-12-10T18:51:40.194Z 4 080-099 36 36 36 57 57 57 F F F
46472023-12-10T18:51:40.195Z 5 100-119 36 36 36 57 57 57 F F F
46482023-12-10T18:51:40.196Z 6 120-139 31 31 31 53 53 53 F F F
46492023-12-10T18:51:40.196Z 7 140-159 37 37 37 59 59 59 F F F
46502023-12-10T18:51:40.196Z 8 160-179 37 37 37 59 59 59 F F F
46512023-12-10T18:51:40.196Z 9 180-199 36 36 36 57 57 57 F F F
46522023-12-10T18:51:40.196Z 10 200-219 33 33 33 55 55 55 F F F
46532023-12-10T18:51:40.196Z 11 220-239 31 31 31 53 53 53 F F F
46542023-12-10T18:51:40.196Z 12 240-259 32 32 32 54 54 54 F F F
46552023-12-10T18:51:40.196Z 13 260-279 37 37 37 59 59 59 F F F
46562023-12-10T18:51:40.197Z 14 280-299 37 37 37 59 59 59 F F F
46572023-12-10T18:51:40.197Z 15 300-319 37 37 37 59 59 59 F F F
46582023-12-10T18:51:40.197Z 16 320-339 32 32 32 54 54 54 F F F
46592023-12-10T18:51:40.197Z 17 340-359 37 37 37 59 59 59 F F F
46602023-12-10T18:51:40.197Z 18 360-379 33 33 33 55 55 55 F F F
46612023-12-10T18:51:40.197Z 19 380-399 26 26 26 46 46 46 F F F
46622023-12-10T18:51:40.198Z 20 400-419 33 33 33 55 55 55 F F F
46632023-12-10T18:51:40.198Z 21 420-439 33 33 33 55 55 55 F F F
46642023-12-10T18:51:40.198Z 22 440-459 37 37 37 59 59 59 F F F
46652023-12-10T18:51:40.198Z 23 460-479 32 32 32 54 54 54 F F F
46662023-12-10T18:51:40.198Z 24 480-499 36 36 36 58 58 58 F F F
46672023-12-10T18:51:40.198Z 25 500-519 31 31 31 53 53 53 F F F
46682023-12-10T18:51:40.198Z 26 520-539 36 36 36 57 57 57 F F F
46692023-12-10T18:51:40.198Z 27 540-559 36 36 36 58 58 58 F F F
46702023-12-10T18:51:40.198Z 28 560-579 32 32 32 54 54 54 F F F
46712023-12-10T18:51:40.199Z 29 580-599 33 33 33 55 55 55 F F F
46722023-12-10T18:51:40.200ZMax gen: 37, Max flush: 59
46732023-12-10T18:51:40.200Z
46742023-12-10T18:51:40.200ZBegin loop 19
46752023-12-10T18:51:40.200ZDownstairs to restart: 2
46762023-12-10T18:51:40.281Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.203971095Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314}
46772023-12-10T18:51:40.281Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.204422005Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314}
46782023-12-10T18:51:40.281Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.20449071Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314}
46792023-12-10T18:51:40.281Z{"msg":"Crucible c840891d-1bed-4dad-90fd-3f7805370327 has session id: 41c68000-44d6-4380-8577-e702d6f65517","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.204523005Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
46802023-12-10T18:51:40.281Z{"msg":"Upstairs opts: Upstairs UUID: c840891d-1bed-4dad-90fd-3f7805370327, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.204581406Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
46812023-12-10T18:51:40.281Z{"msg":"Crucible stats registered with UUID: c840891d-1bed-4dad-90fd-3f7805370327","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.204653081Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
46822023-12-10T18:51:40.281ZCrucible runtime is spawned
46832023-12-10T18:51:40.282ZThe guest has requested activation with gen:38
46842023-12-10T18:51:40.282Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.204891106Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"looper":"2","session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
46852023-12-10T18:51:40.282Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.204955403Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517","task":"up_listen"}
46862023-12-10T18:51:40.282Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.204995587Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
46872023-12-10T18:51:40.282Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.205057693Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
46882023-12-10T18:51:40.282Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.205173884Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"looper":"0","session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
46892023-12-10T18:51:40.282Z{"msg":"[2] c840891d-1bed-4dad-90fd-3f7805370327 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.205248465Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"looper":"2","session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
46902023-12-10T18:51:40.282Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.20529371Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"looper":"1","session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
46912023-12-10T18:51:40.282Z{"msg":"[0] c840891d-1bed-4dad-90fd-3f7805370327 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.20533849Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"looper":"0","session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
46922023-12-10T18:51:40.282Z{"msg":"[1] c840891d-1bed-4dad-90fd-3f7805370327 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.205413308Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"looper":"1","session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
46932023-12-10T18:51:40.297Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.205940937Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
46942023-12-10T18:51:40.297Z{"msg":"c840891d-1bed-4dad-90fd-3f7805370327 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.20600928Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
46952023-12-10T18:51:40.298Z{"msg":"Set desired generation to :38","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.206051943Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
46962023-12-10T18:51:40.298Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.206213878Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
46972023-12-10T18:51:40.298Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.206402771Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
46982023-12-10T18:51:40.298Z{"msg":"[0] c840891d-1bed-4dad-90fd-3f7805370327 (41c68000-44d6-4380-8577-e702d6f65517) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.206752312Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
46992023-12-10T18:51:40.298Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.206833862Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47002023-12-10T18:51:40.299Z{"msg":"[0] client is_active_req TRUE, promote! session 41c68000-44d6-4380-8577-e702d6f65517","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.206875942Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47012023-12-10T18:51:40.299Z{"msg":"[1] c840891d-1bed-4dad-90fd-3f7805370327 (41c68000-44d6-4380-8577-e702d6f65517) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.206949509Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47022023-12-10T18:51:40.299Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.207037548Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47032023-12-10T18:51:40.299Z{"msg":"[1] client is_active_req TRUE, promote! session 41c68000-44d6-4380-8577-e702d6f65517","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.207079029Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47042023-12-10T18:51:40.299Z{"msg":"[2] c840891d-1bed-4dad-90fd-3f7805370327 (41c68000-44d6-4380-8577-e702d6f65517) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.207458977Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47052023-12-10T18:51:40.300Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.207538883Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47062023-12-10T18:51:40.301Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.207627265Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47072023-12-10T18:51:40.301Z{"msg":"[2] client is_active_req TRUE, promote! session 41c68000-44d6-4380-8577-e702d6f65517","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.207654123Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47082023-12-10T18:51:40.301Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.207679896Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47092023-12-10T18:51:40.301Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.207702309Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47102023-12-10T18:51:40.301Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.207767681Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47112023-12-10T18:51:40.301Z{"msg":"c840891d-1bed-4dad-90fd-3f7805370327 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.207794604Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47122023-12-10T18:51:40.301Z{"msg":"c840891d-1bed-4dad-90fd-3f7805370327 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.20785174Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47132023-12-10T18:51:40.301Z{"msg":"[0] c840891d-1bed-4dad-90fd-3f7805370327 (41c68000-44d6-4380-8577-e702d6f65517) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.208299756Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47142023-12-10T18:51:40.302Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.208380824Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47152023-12-10T18:51:40.302Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:40.208411157Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47162023-12-10T18:51:40.302Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.208454319Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47172023-12-10T18:51:40.303Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.208477756Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47182023-12-10T18:51:40.303Z{"msg":"c840891d-1bed-4dad-90fd-3f7805370327 WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.208498884Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47192023-12-10T18:51:40.303Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.20859311Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47202023-12-10T18:51:40.303Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.208650307Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47212023-12-10T18:51:40.303Z{"msg":"[1] c840891d-1bed-4dad-90fd-3f7805370327 (41c68000-44d6-4380-8577-e702d6f65517) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.208696847Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47222023-12-10T18:51:40.303Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.208720363Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47232023-12-10T18:51:40.303Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:40.208764918Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47242023-12-10T18:51:40.303Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.208787232Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47252023-12-10T18:51:40.303Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.208807956Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47262023-12-10T18:51:40.303Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.20882905Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47272023-12-10T18:51:40.303Z{"msg":"c840891d-1bed-4dad-90fd-3f7805370327 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.208849896Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47282023-12-10T18:51:40.303Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.208874629Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47292023-12-10T18:51:40.304Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.208895361Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47302023-12-10T18:51:40.304Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.208914968Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47312023-12-10T18:51:40.304Z{"msg":"c840891d-1bed-4dad-90fd-3f7805370327 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.208935929Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47322023-12-10T18:51:40.304Z{"msg":"[2] c840891d-1bed-4dad-90fd-3f7805370327 (41c68000-44d6-4380-8577-e702d6f65517) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.209485803Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47332023-12-10T18:51:40.304Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.209565425Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47342023-12-10T18:51:40.304Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:40.209589688Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47352023-12-10T18:51:40.304Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.209610012Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47362023-12-10T18:51:40.304Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.209669805Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47372023-12-10T18:51:40.304Z{"msg":"c840891d-1bed-4dad-90fd-3f7805370327 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.209696306Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47382023-12-10T18:51:40.304Z{"msg":"[0]R flush_numbers[0..12]: [57, 54, 59, 55, 57, 57, 53, 59, 59, 57, 55, 53]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.209729746Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47392023-12-10T18:51:40.304Z{"msg":"[0]R generation[0..12]: [36, 32, 37, 33, 36, 36, 31, 37, 37, 36, 33, 31]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.209783183Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47402023-12-10T18:51:40.304Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.209808948Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47412023-12-10T18:51:40.304Z{"msg":"[1]R flush_numbers[0..12]: [57, 54, 59, 55, 57, 57, 53, 59, 59, 57, 55, 53]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.209835233Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47422023-12-10T18:51:40.305Z{"msg":"[1]R generation[0..12]: [36, 32, 37, 33, 36, 36, 31, 37, 37, 36, 33, 31]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.209910171Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47432023-12-10T18:51:40.305Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.209941109Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47442023-12-10T18:51:40.305Z{"msg":"[2]R flush_numbers[0..12]: [57, 54, 59, 55, 57, 57, 53, 59, 59, 57, 55, 53]","v":0,"name":The guest has finished waiting for activation with:38
47452023-12-10T18:51:40.305ZWait for a query_work_queue command to finish before sending IO
47462023-12-10T18:51:40.306Z"crucible","level":30,"time":"2023-12-10T18:51:40.210219145Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47472023-12-10T18:51:40.306Z{"msg":"[2]R generation[0..12]: [36, 32, 37, 33, 36, 36, 31, 37, 37, 36, 33, 31]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.210284101Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47482023-12-10T18:51:40.306Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.210311637Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47492023-12-10T18:51:40.306Z{"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.210352902Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47502023-12-10T18:51:40.306Z{"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.210375644Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47512023-12-10T18:51:40.306Z{"msg":"Next flush: 60","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.210419842Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47522023-12-10T18:51:40.306Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.210441891Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47532023-12-10T18:51:40.306Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.210462122Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47542023-12-10T18:51:40.306Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.210502079Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47552023-12-10T18:51:40.306Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.210524262Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47562023-12-10T18:51:40.306Z{"msg":"c840891d-1bed-4dad-90fd-3f7805370327 is now active with session: 41c68000-44d6-4380-8577-e702d6f65517","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.210547349Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47572023-12-10T18:51:40.306Z{"msg":"c840891d-1bed-4dad-90fd-3f7805370327 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.210593609Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47582023-12-10T18:51:40.306Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.210616269Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47592023-12-10T18:51:40.307Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.210636124Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47602023-12-10T18:51:40.307Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.210655977Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47612023-12-10T18:51:40.307Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.210675664Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47622023-12-10T18:51:40.307Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.21071492Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47632023-12-10T18:51:40.307Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.210737195Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47642023-12-10T18:51:40.307Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.210758485Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47652023-12-10T18:51:40.307Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.210797403Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47662023-12-10T18:51:40.307Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.210818986Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47672023-12-10T18:51:40.307Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.210838603Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47682023-12-10T18:51:40.307Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:40.210877615Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1314,"session_id":"41c68000-44d6-4380-8577-e702d6f65517"}
47692023-12-10T18:51:40.307ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
47702023-12-10T18:51:40.307ZLoading write count information from file "/tmp/test_repair_verify.data"
47712023-12-10T18:51:40.308ZRun Repair workload
47722023-12-10T18:51:40.308Z01/30 Write block 197 len 36864 data: 6 5 5 4 4 4 4 4 5
47732023-12-10T18:51:40.308Z02/30 Read block 66 len 8192
47742023-12-10T18:51:40.308Z03/30 Write block 299 len 32768 data: 5 3 3 4 5 6 4 4
47752023-12-10T18:51:40.308Z04/30 Read block 464 len 16384
47762023-12-10T18:51:40.308Z05/30 Write block 221 len 20480 data: 6 6 5 5 4
47772023-12-10T18:51:40.308Z06/30 Write block 298 len 4096 data: 5
47782023-12-10T18:51:40.308Z07/30 Write block 226 len 36864 data: 4 4 4 4 4 4 4 3 5
47792023-12-10T18:51:40.308Z08/30 Flush
47802023-12-10T18:51:40.308Z09/30 Write block 584 len 4096 data: 2
47812023-12-10T18:51:40.308Z10/30 Read block 285 len 28672
47822023-12-10T18:51:40.308Z11/30 Read block 511 len 28672
47832023-12-10T18:51:40.309Z12/30 Read block 549 len 28672
47842023-12-10T18:51:40.309Z13/30 Flush
47852023-12-10T18:51:40.309Z14/30 Write block 470 len 12288 data: 5 6 4
47862023-12-10T18:51:40.309Z15/30 Write block 148 len 16384 data: 5 5 4 3
47872023-12-10T18:51:40.309Z16/30 Write block 401 len 32768 data: 3 3 3 3 3 3 4 4
47882023-12-10T18:51:40.309Z17/30 Read block 280 len 36864
47892023-12-10T18:51:40.309Z18/30 Write block 500 len 20480 data: 4 4 4 5 5
47902023-12-10T18:51:40.309Z19/30 Read block 437 len 32768
47912023-12-10T18:51:40.309Z20/30 Write block 73 len 28672 data: 3 5 5 5 5 5 6
47922023-12-10T18:51:40.309Z21/30 Read block 521 len 40960
47932023-12-10T18:51:40.420Z22/30 Flush
47942023-12-10T18:51:40.420Z23/30 Write block 514 len 28672 data: 4 4 5 5 4 5 5
47952023-12-10T18:51:40.421Z24/30 Flush
47962023-12-10T18:51:40.421Z25/30 Write block 400 len 24576 data: 4 4 4 4 4 4
47972023-12-10T18:51:40.421Z26/30 Flush
47982023-12-10T18:51:40.421Z27/30 Write block 203 len 4096 data: 5
47992023-12-10T18:51:40.421Z28/30 Write block 465 len 36864 data: 5 5 6 6 7 6 7 5 4
48002023-12-10T18:51:40.421Z29/30 Read block 408 len 12288
48012023-12-10T18:51:40.423Z30/30 Read block 122 len 28672
48022023-12-10T18:51:40.423Z----------------------------------------------------------------
48032023-12-10T18:51:40.423Z Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:30
48042023-12-10T18:51:40.423ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
48052023-12-10T18:51:40.424Z 1 Acked 1000 Write 9 Done Done Sent false
48062023-12-10T18:51:40.424Z 2 Acked 1001 Read 2 Done Done Sent false
48072023-12-10T18:51:40.424Z 3 Acked 1002 Write 8 Done Done Sent false
48082023-12-10T18:51:40.424Z 4 Acked 1003 Read 4 Done Done Sent false
48092023-12-10T18:51:40.424Z 5 Acked 1004 Write 5 Done Done Sent false
48102023-12-10T18:51:40.424Z 6 Acked 1005 Write 1 Done Done Sent false
48112023-12-10T18:51:40.425Z 7 Acked 1006 Write 9 Done Done Sent false
48122023-12-10T18:51:40.425Z 8 Acked 1007 Flush 0 Done Done Sent false
48132023-12-10T18:51:40.425Z 9 Acked 1008 Write 1 Done Done Sent false
48142023-12-10T18:51:40.425Z 10 Acked 1009 Read 7 Done Done Sent false
48152023-12-10T18:51:40.425Z 11 Acked 1010 Read 7 Done Done Sent false
48162023-12-10T18:51:40.425Z 12 Acked 1011 Read 7 Done Done Sent false
48172023-12-10T18:51:40.425Z 13 Acked 1012 Flush 0 Done Done Sent false
48182023-12-10T18:51:40.426Z 14 Acked 1013 Write 3 Done Done Sent false
48192023-12-10T18:51:40.426Z 15 Acked 1014 Write 4 Done Done Sent false
48202023-12-10T18:51:40.426Z 16 Acked 1015 Write 8 Done Done Sent false
48212023-12-10T18:51:40.426Z 17 Acked 1016 Read 9 Done Done Sent false
48222023-12-10T18:51:40.426Z 18 Acked 1017 Write 5 Done Done Sent false
48232023-12-10T18:51:40.426Z 19 Acked 1018 Read 8 Done Done Sent false
48242023-12-10T18:51:40.426Z 20 Acked 1019 Write 7 Done Done Sent false
48252023-12-10T18:51:40.426Z 21 Acked 1020 Read 10 Done Done Sent false
48262023-12-10T18:51:40.426Z 22 Acked 1021 Flush 0 Done Done Sent false
48272023-12-10T18:51:40.427Z 23 Acked 1022 Write 7 Done Done Sent false
48282023-12-10T18:51:40.427Z 24 Acked 1023 Flush 0 Done Done Sent false
48292023-12-10T18:51:40.427Z 25 Acked 1024 Write 6 Done Done Sent false
48302023-12-10T18:51:40.427Z 26 Acked 1025 Flush 0 Done Done Sent false
48312023-12-10T18:51:40.427Z 27 Acked 1026 Write 1 Done Done Sent false
48322023-12-10T18:51:40.428Z 28 Acked 1027 Write 9 Done Done Sent false
48332023-12-10T18:51:40.428Z 29 Acked 1028 Read 3 Done Done Sent false
48342023-12-10T18:51:40.428Z 30 Acked 1029 Read 7 Done Sent Sent false
48352023-12-10T18:51:40.429Z STATES DS:0 DS:1 DS:2 TOTAL
48362023-12-10T18:51:40.429Z New 0 0 0 0
48372023-12-10T18:51:40.429Z Sent 0 1 30 31
48382023-12-10T18:51:40.429Z Done 30 29 0 59
48392023-12-10T18:51:40.429Z Skipped 0 0 0 0
48402023-12-10T18:51:40.430Z Error 0 0 0 0
48412023-12-10T18:51:40.430ZLast Flush: 1025 1025 0
48422023-12-10T18:51:40.431ZDownstairs last five completed:
48432023-12-10T18:51:40.431ZUpstairs last five completed: 30 29 28 27 26 25
48442023-12-10T18:51:40.431ZWrote out file "/tmp/test_repair_verify.data"
48452023-12-10T18:51:40.431Z
48462023-12-10T18:51:40.431ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
48472023-12-10T18:51:40.431ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
48482023-12-10T18:51:40.431Z 0 000-019 36 36 36 57 57 57 F F F
48492023-12-10T18:51:40.432Z 1 020-039 32 32 32 54 54 54 F F F
48502023-12-10T18:51:40.432Z 2 040-059 37 37 37 59 59 59 F F F
48512023-12-10T18:51:40.432Z 3 060-079 38 38 33 62 62 55 F F F <---
48522023-12-10T18:51:40.432Z 4 080-099 36 36 36 57 57 57 F F F
48532023-12-10T18:51:40.432Z 5 100-119 36 36 36 57 57 57 F F F
48542023-12-10T18:51:40.432Z 6 120-139 31 31 31 53 53 53 F F F
48552023-12-10T18:51:40.432Z 7 140-159 38 38 37 62 62 59 F F F <---
48562023-12-10T18:51:40.432Z 8 160-179 37 37 37 59 59 59 F F F
48572023-12-10T18:51:40.432Z 9 180-199 38 38 36 60 60 57 F F F <---
48582023-12-10T18:51:40.433Z 10 200-219 38 38 33 60 60 55 T T F <---
48592023-12-10T18:51:40.433Z 11 220-239 38 38 31 60 60 53 F F F <---
48602023-12-10T18:51:40.433Z 12 240-259 32 32 32 54 54 54 F F F
48612023-12-10T18:51:40.433Z 13 260-279 37 37 37 59 59 59 F F F
48622023-12-10T18:51:40.434Z 14 280-299 38 38 37 60 60 59 F F F <---
48632023-12-10T18:51:40.434Z 15 300-319 38 38 37 60 60 59 F F F <---
48642023-12-10T18:51:40.434Z 16 320-339 32 32 32 54 54 54 F F F
48652023-12-10T18:51:40.435Z 17 340-359 37 37 37 59 59 59 F F F
48662023-12-10T18:51:40.435Z 18 360-379 33 33 33 55 55 55 F F F
48672023-12-10T18:51:40.435Z 19 380-399 26 26 26 46 46 46 F F F
48682023-12-10T18:51:40.435Z 20 400-419 38 38 33 64 64 55 F F F <---
48692023-12-10T18:51:40.435Z 21 420-439 33 33 33 55 55 55 F F F
48702023-12-10T18:51:40.435Z 22 440-459 37 37 37 59 59 59 F F F
48712023-12-10T18:51:40.435Z 23 460-479 38 38 32 62 62 54 T T F <---
48722023-12-10T18:51:40.435Z 24 480-499 36 36 36 58 58 58 F F F
48732023-12-10T18:51:40.435Z 25 500-519 38 38 31 63 63 53 F F F <---
48742023-12-10T18:51:40.436Z 26 520-539 38 38 36 63 63 57 F F F <---
48752023-12-10T18:51:40.436Z 27 540-559 36 36 36 58 58 58 F F F
48762023-12-10T18:51:40.436Z 28 560-579 32 32 32 54 54 54 F F F
48772023-12-10T18:51:40.436Z 29 580-599 38 38 33 61 61 55 F F F <---
48782023-12-10T18:51:40.436ZMax gen: 38, Max flush: 64
48792023-12-10T18:51:40.436ZError: Difference in extent metadata found!
48802023-12-10T18:51:40.436ZA Difference in extent metadata is expected here
48812023-12-10T18:51:40.436ZOn loop 19
48822023-12-10T18:51:40.436Z
48832023-12-10T18:51:40.437ZVerifying data now
48842023-12-10T18:51:45.487ZNo change after verify
48852023-12-10T18:51:45.489ZLoop: 19 Downstairs dump after verify (and repair):
48862023-12-10T18:51:45.508ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
48872023-12-10T18:51:45.508Z 0 000-019 36 36 36 57 57 57 F F F
48882023-12-10T18:51:45.509Z 1 020-039 32 32 32 54 54 54 F F F
48892023-12-10T18:51:45.509Z 2 040-059 37 37 37 59 59 59 F F F
48902023-12-10T18:51:45.509Z 3 060-079 38 38 38 62 62 62 F F F
48912023-12-10T18:51:45.509Z 4 080-099 36 36 36 57 57 57 F F F
48922023-12-10T18:51:45.512Z 5 100-119 36 36 36 57 57 57 F F F
48932023-12-10T18:51:45.512Z 6 120-139 31 31 31 53 53 53 F F F
48942023-12-10T18:51:45.512Z 7 140-159 38 38 38 62 62 62 F F F
48952023-12-10T18:51:45.512Z 8 160-179 37 37 37 59 59 59 F F F
48962023-12-10T18:51:45.512Z 9 180-199 38 38 38 60 60 60 F F F
48972023-12-10T18:51:45.512Z 10 200-219 39 39 39 65 65 65 F F F
48982023-12-10T18:51:45.512Z 11 220-239 38 38 38 60 60 60 F F F
48992023-12-10T18:51:45.513Z 12 240-259 32 32 32 54 54 54 F F F
49002023-12-10T18:51:45.513Z 13 260-279 37 37 37 59 59 59 F F F
49012023-12-10T18:51:45.513Z 14 280-299 38 38 38 60 60 60 F F F
49022023-12-10T18:51:45.513Z 15 300-319 38 38 38 60 60 60 F F F
49032023-12-10T18:51:45.513Z 16 320-339 32 32 32 54 54 54 F F F
49042023-12-10T18:51:45.513Z 17 340-359 37 37 37 59 59 59 F F F
49052023-12-10T18:51:45.513Z 18 360-379 33 33 33 55 55 55 F F F
49062023-12-10T18:51:45.513Z 19 380-399 26 26 26 46 46 46 F F F
49072023-12-10T18:51:45.513Z 20 400-419 38 38 38 64 64 64 F F F
49082023-12-10T18:51:45.513Z 21 420-439 33 33 33 55 55 55 F F F
49092023-12-10T18:51:45.513Z 22 440-459 37 37 37 59 59 59 F F F
49102023-12-10T18:51:45.514Z 23 460-479 39 39 39 65 65 65 F F F
49112023-12-10T18:51:45.514Z 24 480-499 36 36 36 58 58 58 F F F
49122023-12-10T18:51:45.514Z 25 500-519 38 38 38 63 63 63 F F F
49132023-12-10T18:51:45.514Z 26 520-539 38 38 38 63 63 63 F F F
49142023-12-10T18:51:45.514Z 27 540-559 36 36 36 58 58 58 F F F
49152023-12-10T18:51:45.514Z 28 560-579 32 32 32 54 54 54 F F F
49162023-12-10T18:51:45.514Z 29 580-599 38 38 38 61 61 61 F F F
49172023-12-10T18:51:45.514ZMax gen: 39, Max flush: 65
49182023-12-10T18:51:45.518Z
49192023-12-10T18:51:45.518ZBegin loop 20
49202023-12-10T18:51:45.518ZDownstairs to restart: 1
49212023-12-10T18:51:45.550Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.54417784Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322}
49222023-12-10T18:51:45.550Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.544599154Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322}
49232023-12-10T18:51:45.550Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.544627953Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322}
49242023-12-10T18:51:45.558Z{"msg":"Crucible 7badbb23-ca2c-4bbf-ba4d-dd7787d2b382 has session id: 67c33ff8-c327-4aee-8a63-d5a846447f46","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.54467256Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49252023-12-10T18:51:45.558Z{"msg":"Upstairs opts: Upstairs UUID: 7badbb23-ca2c-4bbf-ba4d-dd7787d2b382, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
49262023-12-10T18:51:45.558Z,"time":"2023-12-10T18:51:45.544754254Z"The guest has requested activation with gen:40
49272023-12-10T18:51:45.559Z,"hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49282023-12-10T18:51:45.559Z{"msg":"Crucible stats registered with UUID: 7badbb23-ca2c-4bbf-ba4d-dd7787d2b382","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.545042105Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49292023-12-10T18:51:45.559Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.545088501Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46","task":"up_listen"}
49302023-12-10T18:51:45.559Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.545158645Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49312023-12-10T18:51:45.559Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.545198041Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49322023-12-10T18:51:45.559Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.545233094Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"looper":"1","session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49332023-12-10T18:51:45.560Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.54541952Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"looper":"0","session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49342023-12-10T18:51:45.560Z{"msg":"[1] 7badbb23-ca2c-4bbf-ba4d-dd7787d2b382 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.545501067Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"looper":"1","session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49352023-12-10T18:51:45.560Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.545531698Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"looper":"2","session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49362023-12-10T18:51:45.560Z{"msg":"[0] 7badbb23-ca2c-4bbf-ba4d-dd7787d2b382 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.545653306Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"looper":"0","session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49372023-12-10T18:51:45.560Z{"msg":"[2] 7badbb23-ca2c-4bbf-ba4d-dd7787d2b382 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.545745949Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"looper":"2","session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49382023-12-10T18:51:45.561Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.546211784Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49392023-12-10T18:51:45.561Z{"msg":"7badbb23-ca2c-4bbf-ba4d-dd7787d2b382 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.546331995Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49402023-12-10T18:51:45.561Z{"msg":"Set desired generation to :40","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.546375331Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49412023-12-10T18:51:45.561Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.546609165Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49422023-12-10T18:51:45.561Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.546693425Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49432023-12-10T18:51:45.561Z{"msg":"[0] 7badbb23-ca2c-4bbf-ba4d-dd7787d2b382 (67c33ff8-c327-4aee-8a63-d5a846447f46) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.547120118Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49442023-12-10T18:51:45.561Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.547195544Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49452023-12-10T18:51:45.561Z{"msg":"[0] client is_active_req TRUE, promote! session 67c33ff8-c327-4aee-8a63-d5a846447f46","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.547272596Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49462023-12-10T18:51:45.561Z{"msg":"[2] 7badbb23-ca2c-4bbf-ba4d-dd7787d2b382 (67c33ff8-c327-4aee-8a63-d5a846447f46) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.547317331Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49472023-12-10T18:51:45.561Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.547471389Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49482023-12-10T18:51:45.562Z{"msg":"[2] client is_active_req TRUE, promote! session 67c33ff8-c327-4aee-8a63-d5a846447f46","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.547499874Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49492023-12-10T18:51:45.562Z{"msg":"[1] 7badbb23-ca2c-4bbf-ba4d-dd7787d2b382 (67c33ff8-c327-4aee-8a63-d5a846447f46) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.547522971Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49502023-12-10T18:51:45.562Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.547607511Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49512023-12-10T18:51:45.562Z{"msg":"[1] client is_active_req TRUE, promote! session 67c33ff8-c327-4aee-8a63-d5a846447f46","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.547684256Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49522023-12-10T18:51:45.562Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.548108875Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49532023-12-10T18:51:45.562Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.548167119Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49542023-12-10T18:51:45.562Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.548262037Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49552023-12-10T18:51:45.562Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.548322507Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49562023-12-10T18:51:45.562Z{"msg":"7badbb23-ca2c-4bbf-ba4d-dd7787d2b382 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.54836515Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49572023-12-10T18:51:45.566Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.548431199Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49582023-12-10T18:51:45.566Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.548470117Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49592023-12-10T18:51:45.566Z{"msg":"7badbb23-ca2c-4bbf-ba4d-dd7787d2b382 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.548537952Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49602023-12-10T18:51:45.566Z{"msg":"7badbb23-ca2c-4bbf-ba4d-dd7787d2b382 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.548582391Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49612023-12-10T18:51:45.566Z{"msg":"[2] 7badbb23-ca2c-4bbf-ba4d-dd7787d2b382 (67c33ff8-c327-4aee-8a63-d5a846447f46) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.548936849Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49622023-12-10T18:51:45.566Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.548996377Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49632023-12-10T18:51:45.566Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:45.549032033Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49642023-12-10T18:51:45.566Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.549064636Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49652023-12-10T18:51:45.566Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.549125731Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49662023-12-10T18:51:45.566Z{"msg":"7badbb23-ca2c-4bbf-ba4d-dd7787d2b382 WaitActive WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.549199673Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49672023-12-10T18:51:45.566Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.549275939Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49682023-12-10T18:51:45.566Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.549307872Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49692023-12-10T18:51:45.567Z{"msg":"[0] 7badbb23-ca2c-4bbf-ba4d-dd7787d2b382 (67c33ff8-c327-4aee-8a63-d5a846447f46) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.549339364Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49702023-12-10T18:51:45.567Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.549407942Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49712023-12-10T18:51:45.567Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:45.549434918Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49722023-12-10T18:51:45.567Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.549479248Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49732023-12-10T18:51:45.567Z{"msg":"[1] 7badbb23-ca2c-4bbf-ba4d-dd7787d2b382 (67c33ff8-c327-4aee-8a63-d5a846447f46) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.54950519Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49742023-12-10T18:51:45.567Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.549561401Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49752023-12-10T18:51:45.567Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:45.54960734Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49762023-12-10T18:51:45.567Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.54963052Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49772023-12-10T18:51:45.567Z{"msg":"7badbb23-ca2c-4bbf-ba4d-dd7787d2b382 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.549676729Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49782023-12-10T18:51:45.567Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.549733729Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49792023-12-10T18:51:45.567Z{"msg":"[0]R flush_numbers[0..12]: [57, 54, 59, 62, 57, 57, 53, 62, 59, 60, 65, 60]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.549757088Z","hostname":"The guest has finished waiting for activation with:40
49802023-12-10T18:51:45.567Zip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"Wait for a query_work_queue command to finish before sending IO
49812023-12-10T18:51:45.567Z67c33ff8-c327-4aee-8a63-d5a846447f46"}
49822023-12-10T18:51:45.567Z{"msg":"[0]R generation[0..12]: [36, 32, 37, 38, 36, 36, 31, 38, 37, 38, 39, 38]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.5497905Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49832023-12-10T18:51:45.567Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.549825406Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49842023-12-10T18:51:45.567Z{"msg":"[1]R flush_numbers[0..12]: [57, 54, 59, 62, 57, 57, 53, 62, 59, 60, 65, 60]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.549861139Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49852023-12-10T18:51:45.567Z{"msg":"[1]R generation[0..12]: [36, 32, 37, 38, 36, 36, 31, 38, 37, 38, 39, 38]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.549898639Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49862023-12-10T18:51:45.567Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.549934934Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49872023-12-10T18:51:45.568Z{"msg":"[2]R flush_numbers[0..12]: [57, 54, 59, 62, 57, 57, 53, 62, 59, 60, 65, 60]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.549973569Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49882023-12-10T18:51:45.570Z{"msg":"[2]R generation[0..12]: [36, 32, 37, 38, 36, 36, 31, 38, 37, 38, 39, 38]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.550011541Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49892023-12-10T18:51:45.570Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.55007256Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49902023-12-10T18:51:45.570Z{"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.550111355Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49912023-12-10T18:51:45.570Z{"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.5501707Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49922023-12-10T18:51:45.570Z{"msg":"Next flush: 66","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.550208544Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49932023-12-10T18:51:45.570Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.550281079Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49942023-12-10T18:51:45.570Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.550319975Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49952023-12-10T18:51:45.570Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.550375732Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49962023-12-10T18:51:45.570Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.550414225Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49972023-12-10T18:51:45.570Z{"msg":"7badbb23-ca2c-4bbf-ba4d-dd7787d2b382 is now active with session: 67c33ff8-c327-4aee-8a63-d5a846447f46","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.550473121Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49982023-12-10T18:51:45.570Z{"msg":"7badbb23-ca2c-4bbf-ba4d-dd7787d2b382 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.550512288Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
49992023-12-10T18:51:45.573Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.550599495Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
50002023-12-10T18:51:45.573Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.550637836Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
50012023-12-10T18:51:45.573Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.550701764Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
50022023-12-10T18:51:45.573Z{"msg":"7badbb23-ca2c-4bbf-ba4d-dd7787d2b382 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.550741399Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
50032023-12-10T18:51:45.573Z{"msg":"Set check for repair","v":0,"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
50042023-12-10T18:51:45.573Zname":"crucible","level":30,"time":"2023-12-10T18:51:45.550848522Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
50052023-12-10T18:51:45.573Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.550903455Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
50062023-12-10T18:51:45.573Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.550942215Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
50072023-12-10T18:51:45.573Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.550995436Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
50082023-12-10T18:51:45.573Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.551033822Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
50092023-12-10T18:51:45.573Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.551071067Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
50102023-12-10T18:51:45.573Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.551108307Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
50112023-12-10T18:51:45.574Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.551160125Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
50122023-12-10T18:51:45.574Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.551198055Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
50132023-12-10T18:51:45.574Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:45.551252558Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1322,"session_id":"67c33ff8-c327-4aee-8a63-d5a846447f46"}
50142023-12-10T18:51:45.574ZLoading write count information from file "/tmp/test_repair_verify.data"
50152023-12-10T18:51:45.574ZRun Repair workload
50162023-12-10T18:51:45.574Z01/30 Write block 374 len 28672 data: 2 4 4 4 5 5 5
50172023-12-10T18:51:45.574Z02/30 Write block 547 len 12288 data: 6 6 3
50182023-12-10T18:51:45.578Z03/30 Read block 249 len 8192
50192023-12-10T18:51:45.578Z04/30 Read block 240 len 20480
50202023-12-10T18:51:45.578Z05/30 Write block 561 len 24576 data: 4 4 3 3 3 3
50212023-12-10T18:51:45.578Z06/30 Read block 434 len 32768
50222023-12-10T18:51:45.578Z07/30 Flush
50232023-12-10T18:51:45.578Z08/30 Write block 206 len 12288 data: 3 3 2
50242023-12-10T18:51:45.578Z09/30 Read block 381 len 36864
50252023-12-10T18:51:45.579Z10/30 Read block 183 len 28672
50262023-12-10T18:51:45.579Z11/30 Read block 576 len 32768
50272023-12-10T18:51:45.579Z12/30 Read block 38 len 4096
50282023-12-10T18:51:45.579Z13/30 Write block 469 len 24576 data: 8 7 8 6 5 4
50292023-12-10T18:51:45.579Z14/30 Flush
50302023-12-10T18:51:45.584Z15/30 Write block 312 len 36864 data: 2 2 3 2 2 2 2 2 2
50312023-12-10T18:51:45.584Z16/30 Read block 15 len 4096
50322023-12-10T18:51:45.586Z17/30 Read block 347 len 12288
50332023-12-10T18:51:45.589Z18/30 Write block 234 len 32768 data: 6 5 5 5 6 8 7 7
50342023-12-10T18:51:45.589Z19/30 Read block 510 len 12288
50352023-12-10T18:51:45.589Z20/30 Write block 253 len 4096 data: 4
50362023-12-10T18:51:45.589Z21/30 Read block 228 len 20480
50372023-12-10T18:51:45.592Z22/30 Read block 19 len 12288
50382023-12-10T18:51:45.592Z23/30 Read block 521 len 8192
50392023-12-10T18:51:45.592Z24/30 Read block 99 len 4096
50402023-12-10T18:51:45.595Z25/30 Write block 187 len 20480 data: 3 3 3 3 4
50412023-12-10T18:51:45.595Z26/30 Write block 14 len 16384 data: 3 3 3 5
50422023-12-10T18:51:45.598Z27/30 Write block 378 len 32768 data: 6 6 6 4 4 4 5 5
50432023-12-10T18:51:45.598Z28/30 Read block 13 len 36864
50442023-12-10T18:51:45.601Z29/30 Write block 211 len 24576 data: 2 3 4 7 7 8
50452023-12-10T18:51:45.601Z30/30 Write block 382 len 24576 data: 5 5 6 6 5 6
50462023-12-10T18:51:45.604Z----------------------------------------------------------------
50472023-12-10T18:51:45.605Z Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:16
50482023-12-10T18:51:45.605ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
50492023-12-10T18:51:45.605Z 15 Acked 1014 Write 9 Done Done Done false
50502023-12-10T18:51:45.605Z 16 Acked 1015 Read 1 Done Done Done false
50512023-12-10T18:51:45.605Z 17 Acked 1016 Read 3 Done Sent Done false
50522023-12-10T18:51:45.605Z 18 Acked 1017 Write 8 Done Sent Done false
50532023-12-10T18:51:45.605Z 19 Acked 1018 Read 3 Done Sent Done false
50542023-12-10T18:51:45.605Z 20 Acked 1019 Write 1 Done Sent Done false
50552023-12-10T18:51:45.605Z 21 Acked 1020 Read 5 Done Sent Done false
50562023-12-10T18:51:45.606Z 22 Acked 1021 Read 3 Done Sent Done false
50572023-12-10T18:51:45.606Z 23 Acked 1022 Read 2 Done Sent Done false
50582023-12-10T18:51:45.606Z 24 Acked 1023 Read 1 Done Sent Done false
50592023-12-10T18:51:45.606Z 25 Acked 1024 Write 5 Done Sent Done false
50602023-12-10T18:51:45.606Z 26 Acked 1025 Write 4 Done Sent Done false
50612023-12-10T18:51:45.606Z 27 Acked 1026 Write 8 Done Sent Done false
50622023-12-10T18:51:45.606Z 28 Acked 1027 Read 9 Done Sent Done false
50632023-12-10T18:51:45.606Z 29 Acked 1028 Write 6 Done Sent Done false
50642023-12-10T18:51:45.606Z 30 Acked 1029 Write 6 Sent Sent Sent false
50652023-12-10T18:51:45.606Z STATES DS:0 DS:1 DS:2 TOTAL
50662023-12-10T18:51:45.606Z New 0 0 0 0
50672023-12-10T18:51:45.606Z Sent 1 14 1 16
50682023-12-10T18:51:45.606Z Done 15 2 15 32
50692023-12-10T18:51:45.606Z Skipped 0 0 0 0
50702023-12-10T18:51:45.606Z Error 0 0 0 0
50712023-12-10T18:51:45.606ZLast Flush: 1013 1013 1013
50722023-12-10T18:51:45.607ZDownstairs last five completed: 1013 1012 1011 1010 1009 1008
50732023-12-10T18:51:45.607ZUpstairs last five completed: 30 29 28 27 26 25
50742023-12-10T18:51:45.607ZWrote out file "/tmp/test_repair_verify.data"
50752023-12-10T18:51:45.610Z
50762023-12-10T18:51:45.613ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
50772023-12-10T18:51:45.645ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
50782023-12-10T18:51:45.645Z 0 000-019 36 36 36 57 57 57 T F T <---
50792023-12-10T18:51:45.660Z 1 020-039 32 32 32 54 54 54 F F F
50802023-12-10T18:51:45.662Z 2 040-059 37 37 37 59 59 59 F F F
50812023-12-10T18:51:45.663Z 3 060-079 38 38 38 62 62 62 F F F
50822023-12-10T18:51:45.663Z 4 080-099 36 36 36 57 57 57 F F F
50832023-12-10T18:51:45.663Z 5 100-119 36 36 36 57 57 57 F F F
50842023-12-10T18:51:45.663Z 6 120-139 31 31 31 53 53 53 F F F
50852023-12-10T18:51:45.663Z 7 140-159 38 38 38 62 62 62 F F F
50862023-12-10T18:51:45.663Z 8 160-179 37 37 37 59 59 59 F F F
50872023-12-10T18:51:45.663Z 9 180-199 38 38 38 60 60 60 T F T <---
50882023-12-10T18:51:45.663Z 10 200-219 40 40 40 67 67 67 T F T <---
50892023-12-10T18:51:45.664Z 11 220-239 38 38 38 60 60 60 T F T <---
50902023-12-10T18:51:45.664Z 12 240-259 32 32 32 54 54 54 T F T <---
50912023-12-10T18:51:45.664Z 13 260-279 37 37 37 59 59 59 F F F
50922023-12-10T18:51:45.664Z 14 280-299 38 38 38 60 60 60 F F F
50932023-12-10T18:51:45.665Z 15 300-319 38 38 38 60 60 60 T T T
50942023-12-10T18:51:45.665Z 16 320-339 32 32 32 54 54 54 T T T
50952023-12-10T18:51:45.665Z 17 340-359 37 37 37 59 59 59 F F F
50962023-12-10T18:51:45.665Z 18 360-379 40 40 40 66 66 66 T F T <---
50972023-12-10T18:51:45.665Z 19 380-399 40 40 40 66 66 66 T F T <---
50982023-12-10T18:51:45.666Z 20 400-419 38 38 38 64 64 64 F F F
50992023-12-10T18:51:45.666Z 21 420-439 33 33 33 55 55 55 F F F
51002023-12-10T18:51:45.666Z 22 440-459 37 37 37 59 59 59 F F F
51012023-12-10T18:51:45.666Z 23 460-479 40 40 40 67 67 67 F F F
51022023-12-10T18:51:45.666Z 24 480-499 36 36 36 58 58 58 F F F
51032023-12-10T18:51:45.666Z 25 500-519 38 38 38 63 63 63 F F F
51042023-12-10T18:51:45.666Z 26 520-539 38 38 38 63 63 63 F F F
51052023-12-10T18:51:45.667Z 27 540-559 40 40 40 66 66 66 F F F
51062023-12-10T18:51:45.667Z 28 560-579 40 40 40 66 66 66 F F F
51072023-12-10T18:51:45.667Z 29 580-599 38 38 38 61 61 61 F F F
51082023-12-10T18:51:45.667ZMax gen: 40, Max flush: 67
51092023-12-10T18:51:45.667ZError: Difference in extent metadata found!
51102023-12-10T18:51:45.667ZA Difference in extent metadata is expected here
51112023-12-10T18:51:45.668ZOn loop 20
51122023-12-10T18:51:45.668Z
51132023-12-10T18:51:45.668ZVerifying data now
51142023-12-10T18:51:51.637ZNo change after verify
51152023-12-10T18:51:51.637ZLoop: 20 Downstairs dump after verify (and repair):
51162023-12-10T18:51:51.661ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51172023-12-10T18:51:51.661Z 0 000-019 41 41 41 68 68 68 F F F
51182023-12-10T18:51:51.670Z 1 020-039 32 32 32 54 54 54 F F F
51192023-12-10T18:51:51.670Z 2 040-059 37 37 37 59 59 59 F F F
51202023-12-10T18:51:51.670Z 3 060-079 38 38 38 62 62 62 F F F
51212023-12-10T18:51:51.670Z 4 080-099 36 36 36 57 57 57 F F F
51222023-12-10T18:51:51.670Z 5 100-119 36 36 36 57 57 57 F F F
51232023-12-10T18:51:51.670Z 6 120-139 31 31 31 53 53 53 F F F
51242023-12-10T18:51:51.670Z 7 140-159 38 38 38 62 62 62 F F F
51252023-12-10T18:51:51.670Z 8 160-179 37 37 37 59 59 59 F F F
51262023-12-10T18:51:51.670Z 9 180-199 41 41 41 68 68 68 F F F
51272023-12-10T18:51:51.670Z 10 200-219 41 41 41 68 68 68 F F F
51282023-12-10T18:51:51.670Z 11 220-239 41 41 41 68 68 68 F F F
51292023-12-10T18:51:51.670Z 12 240-259 41 41 41 68 68 68 F F F
51302023-12-10T18:51:51.670Z 13 260-279 37 37 37 59 59 59 F F F
51312023-12-10T18:51:51.670Z 14 280-299 38 38 38 60 60 60 F F F
51322023-12-10T18:51:51.670Z 15 300-319 41 41 41 68 68 68 F F F
51332023-12-10T18:51:51.671Z 16 320-339 41 41 41 68 68 68 F F F
51342023-12-10T18:51:51.671Z 17 340-359 37 37 37 59 59 59 F F F
51352023-12-10T18:51:51.671Z 18 360-379 41 41 41 68 68 68 F F F
51362023-12-10T18:51:51.671Z 19 380-399 41 41 41 68 68 68 F F F
51372023-12-10T18:51:51.671Z 20 400-419 38 38 38 64 64 64 F F F
51382023-12-10T18:51:51.673Z 21 420-439 33 33 33 55 55 55 F F F
51392023-12-10T18:51:51.674Z 22 440-459 37 37 37 59 59 59 F F F
51402023-12-10T18:51:51.674Z 23 460-479 40 40 40 67 67 67 F F F
51412023-12-10T18:51:51.674Z 24 480-499 36 36 36 58 58 58 F F F
51422023-12-10T18:51:51.674Z 25 500-519 38 38 38 63 63 63 F F F
51432023-12-10T18:51:51.674Z 26 520-539 38 38 38 63 63 63 F F F
51442023-12-10T18:51:51.674Z 27 540-559 40 40 40 66 66 66 F F F
51452023-12-10T18:51:51.674Z 28 560-579 40 40 40 66 66 66 F F F
51462023-12-10T18:51:51.674Z 29 580-599 38 38 38 61 61 61 F F F
51472023-12-10T18:51:51.674ZMax gen: 41, Max flush: 68
51482023-12-10T18:51:51.674Z
51492023-12-10T18:51:51.674ZBegin loop 21
51502023-12-10T18:51:51.674ZDownstairs to restart: 2
51512023-12-10T18:51:51.679Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.678354126Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330}
51522023-12-10T18:51:51.679Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.678852595Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330}
51532023-12-10T18:51:51.698Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.678921135Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330}
51542023-12-10T18:51:51.698Z{"msg":"Crucible a9eeca3c-8a45-4859-9320-fcf6c2918717 has session id: 5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.67899185Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51552023-12-10T18:51:51.699Z{"msg":"Upstairs opts: Upstairs UUID: a9eeca3c-8a45-4859-9320-fcf6c2918717, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.679059215Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51562023-12-10T18:51:51.699Z{"msg":"Crucible stats registered with UUID: a9eeca3c-8a45-4859-9320-fcf6c2918717","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.679098054Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51572023-12-10T18:51:51.699ZCrucible runtime is spawned
51582023-12-10T18:51:51.699ZThe guest has requested activation with gen:42
51592023-12-10T18:51:51.699Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.679374487Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0","task":"up_listen"}
51602023-12-10T18:51:51.699Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.679444938Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51612023-12-10T18:51:51.699Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.679471566Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51622023-12-10T18:51:51.700Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.67953275Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"looper":"0","session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51632023-12-10T18:51:51.700Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.679758007Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"looper":"2","session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51642023-12-10T18:51:51.700Z{"msg":"[0] a9eeca3c-8a45-4859-9320-fcf6c2918717 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.67992126Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"looper":"0","session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51652023-12-10T18:51:51.700Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.680090112Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"looper":"1","session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51662023-12-10T18:51:51.700Z{"msg":"[2] a9eeca3c-8a45-4859-9320-fcf6c2918717 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.68012588Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"looper":"2","session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51672023-12-10T18:51:51.700Z{"msg":"[1] a9eeca3c-8a45-4859-9320-fcf6c2918717 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.680360794Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"looper":"1","session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51682023-12-10T18:51:51.700Z{"msg":"a9eeca3c-8a45-4859-9320-fcf6c2918717 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.680697781Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51692023-12-10T18:51:51.700Z{"msg":"Set desired generation to :42","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.680821254Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51702023-12-10T18:51:51.700Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.680939056Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51712023-12-10T18:51:51.700Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.68127211Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51722023-12-10T18:51:51.700Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.681340319Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51732023-12-10T18:51:51.700Z{"msg":"[0] a9eeca3c-8a45-4859-9320-fcf6c2918717 (5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.681631474Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51742023-12-10T18:51:51.700Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.681692402Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51752023-12-10T18:51:51.700Z{"msg":"[0] client is_active_req TRUE, promote! session 5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.681717645Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51762023-12-10T18:51:51.700Z{"msg":"[1] a9eeca3c-8a45-4859-9320-fcf6c2918717 (5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.681770477Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51772023-12-10T18:51:51.700Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.681850497Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51782023-12-10T18:51:51.700Z{"msg":"[1] client is_active_req TRUE, promote! session 5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.681895738Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51792023-12-10T18:51:51.700Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.682318903Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51802023-12-10T18:51:51.701Z{"msg":"[2] a9eeca3c-8a45-4859-9320-fcf6c2918717 (5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.68248985Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51812023-12-10T18:51:51.701Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.682533433Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51822023-12-10T18:51:51.701Z{"msg":"[2] client is_active_req TRUE, promote! session 5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.682603939Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51832023-12-10T18:51:51.701Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.682652498Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51842023-12-10T18:51:51.701Z{"msg":"a9eeca3c-8a45-4859-9320-fcf6c2918717 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.682720349Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51852023-12-10T18:51:51.701Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.682761302Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51862023-12-10T18:51:51.702Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.682835031Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51872023-12-10T18:51:51.702Z{"msg":"a9eeca3c-8a45-4859-9320-fcf6c2918717 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.682873832Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51882023-12-10T18:51:51.702Z{"msg":"[0] a9eeca3c-8a45-4859-9320-fcf6c2918717 (5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.683219997Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51892023-12-10T18:51:51.702Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.683284635Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51902023-12-10T18:51:51.702Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:51.683310552Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51912023-12-10T18:51:51.702Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.683352604Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51922023-12-10T18:51:51.702Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.683375931Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51932023-12-10T18:51:51.702Z{"msg":"a9eeca3c-8a45-4859-9320-fcf6c2918717 WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.683417625Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51942023-12-10T18:51:51.702Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.683441746Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51952023-12-10T18:51:51.702Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.683461962Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51962023-12-10T18:51:51.702Z{"msg":"[1] a9eeca3c-8a45-4859-9320-fcf6c2918717 (5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.683501694Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51972023-12-10T18:51:51.702Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.683567646Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51982023-12-10T18:51:51.702Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:51.683607157Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
51992023-12-10T18:51:51.711Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.683743368Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52002023-12-10T18:51:51.711Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.68380105Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52012023-12-10T18:51:51.711Z{"msg":"a9eeca3c-8a45-4859-9320-fcf6c2918717 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.683826281Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52022023-12-10T18:51:51.711Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.683846954Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52032023-12-10T18:51:51.711Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.683876242Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52042023-12-10T18:51:51.711Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.683912068Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52052023-12-10T18:51:51.711Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.683948852Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52062023-12-10T18:51:51.711Z{"msg":"a9eeca3c-8a45-4859-9320-fcf6c2918717 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.683979828Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52072023-12-10T18:51:51.712Z{"msg":"[2] a9eeca3c-8a45-4859-9320-fcf6c2918717 (5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.684413971Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52082023-12-10T18:51:51.712Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.684469963Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52092023-12-10T18:51:51.712Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:51.684493721Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52102023-12-10T18:51:51.712Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.684580725Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52112023-12-10T18:51:51.713Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.684608148Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52122023-12-10T18:51:51.713Z{"msg":"a9eeca3c-8a45-4859-9320-fcf6c2918717 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.684670963Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52132023-12-10T18:51:51.713Z{The guest has finished waiting for activation with:42
52142023-12-10T18:51:51.713Z"msg":"[0]R flush_numbers[0..12]: [68, 54, 59, 62, 57, 57, 53, 62, 59, 68, 68, 68]","v":0,"name":"Wait for a query_work_queue command to finish before sending IO
52152023-12-10T18:51:51.713Zcrucible","level":30,"time":"2023-12-10T18:51:51.684704759Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52162023-12-10T18:51:51.713Z{"msg":"[0]R generation[0..12]: [41, 32, 37, 38, 36, 36, 31, 38, 37, 41, 41, 41]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.68480226Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52172023-12-10T18:51:51.713Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.684829646Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52182023-12-10T18:51:51.713Z{"msg":"[1]R flush_numbers[0..12]: [68, 54, 59, 62, 57, 57, 53, 62, 59, 68, 68, 68]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.684858446Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52192023-12-10T18:51:51.713Z{"msg":"[1]R generation[0..12]: [41, 32, 37, 38, 36, 36, 31, 38, 37, 41, 41, 41]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.684923354Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52202023-12-10T18:51:51.713Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.684964837Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52212023-12-10T18:51:51.713Z{"msg":"[2]R flush_numbers[0..12]: [68, 54, 59, 62, 57, 57, 53, 62, 59, 68, 68, 68]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.685006519Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52222023-12-10T18:51:51.713Z{"msg":"[2]R generation[0..12]: [41, 32, 37, 38, 36, 36, 31, 38, 37, 41, 41, 41]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.685031365Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52232023-12-10T18:51:51.713Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.685059058Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52242023-12-10T18:51:51.713Z{"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.685080231Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52252023-12-10T18:51:51.713Z{"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.68510043Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52262023-12-10T18:51:51.713Z{"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.68521814Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52272023-12-10T18:51:51.713Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.685258774Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52282023-12-10T18:51:51.713Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.685314729Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52292023-12-10T18:51:51.714Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.685352839Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52302023-12-10T18:51:51.714Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.685409388Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52312023-12-10T18:51:51.714Z{"msg":"a9eeca3c-8a45-4859-9320-fcf6c2918717 is now active with session: 5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.685450579Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52322023-12-10T18:51:51.714Z{"msg":"a9eeca3c-8a45-4859-9320-fcf6c2918717 Set Active after no repair","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
52332023-12-10T18:51:51.714Z,"time":"2023-12-10T18:51:51.685514835Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52342023-12-10T18:51:51.714Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.6856017Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52352023-12-10T18:51:51.714Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.6856338Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52362023-12-10T18:51:51.715Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.685654531Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52372023-12-10T18:51:51.715Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.685680953Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52382023-12-10T18:51:51.715Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.685702181Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52392023-12-10T18:51:51.715Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.685722102Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52402023-12-10T18:51:51.715Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.685742048Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52412023-12-10T18:51:51.715Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.685762476Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52422023-12-10T18:51:51.715Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.685810282Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52432023-12-10T18:51:51.715Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.685834019Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52442023-12-10T18:51:51.715Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:51.685855438Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1330,"session_id":"5c5bae95-cc9b-4a4b-bc52-cc8408ad12b0"}
52452023-12-10T18:51:51.715ZLoading write count information from file "/tmp/test_repair_verify.data"
52462023-12-10T18:51:51.715ZRun Repair workload
52472023-12-10T18:51:51.715Z01/30 Write block 166 len 12288 data: 9 9 6
52482023-12-10T18:51:51.715Z02/30 Read block 530 len 4096
52492023-12-10T18:51:51.715Z03/30 Read block 543 len 16384
52502023-12-10T18:51:51.715Z04/30 Write block 191 len 20480 data: 5 5 5 5 7
52512023-12-10T18:51:51.715Z05/30 Read block 417 len 24576
52522023-12-10T18:51:51.715Z06/30 Read block 354 len 32768
52532023-12-10T18:51:51.715Z07/30 Read block 472 len 20480
52542023-12-10T18:51:51.716Z08/30 Write block 539 len 24576 data: 4 7 9 10 9 8
52552023-12-10T18:51:51.716Z09/30 Read block 362 len 4096
52562023-12-10T18:51:51.716Z10/30 Flush
52572023-12-10T18:51:51.716Z11/30 Write block 319 len 24576 data: 3 3 3 4 3 3
52582023-12-10T18:51:51.716Z12/30 Read block 396 len 8192
52592023-12-10T18:51:51.719Z13/30 Read block 497 len 16384
52602023-12-10T18:51:51.720Z14/30 Write block 406 len 20480 data: 4 5 5 3 4
52612023-12-10T18:51:51.720Z15/30 Flush
52622023-12-10T18:51:51.720Z16/30 Write block 39 len 28672 data: 4 4 4 4 4 4 5
52632023-12-10T18:51:51.720Z17/30 Read block 467 len 4096
52642023-12-10T18:51:51.720Z18/30 Read block 387 len 40960
52652023-12-10T18:51:51.720Z19/30 Read block 116 len 16384
52662023-12-10T18:51:51.720Z20/30 Flush
52672023-12-10T18:51:51.723Z21/30 Write block 361 len 12288 data: 4 4 4
52682023-12-10T18:51:51.723Z22/30 Write block 237 len 32768 data: 6 7 9 8 8 8 8 8
52692023-12-10T18:51:51.727Z23/30 Write block 322 len 20480 data: 5 4 4 3 3
52702023-12-10T18:51:51.727Z24/30 Read block 166 len 4096
52712023-12-10T18:51:51.727Z25/30 Flush
52722023-12-10T18:51:51.733Z26/30 Write block 164 len 16384 data: 8 9 10 10
52732023-12-10T18:51:51.742Z27/30 Read block 584 len 8192
52742023-12-10T18:51:51.742Z28/30 Read block 499 len 28672
52752023-12-10T18:51:51.742Z29/30 Write block 181 len 12288 data: 3 4 4
52762023-12-10T18:51:51.742Z30/30 Read block 322 len 12288
52772023-12-10T18:51:51.814Z----------------------------------------------------------------
52782023-12-10T18:51:51.814Z Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30
52792023-12-10T18:51:51.814ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
52802023-12-10T18:51:51.815Z 1 Acked 1000 Write 3 Done Done Done false
52812023-12-10T18:51:51.815Z 2 Acked 1001 Read 1 Done Done Done false
52822023-12-10T18:51:51.815Z 3 Acked 1002 Read 4 Done Done Done false
52832023-12-10T18:51:51.815Z 4 Acked 1003 Write 5 Done Done Done false
52842023-12-10T18:51:51.815Z 5 Acked 1004 Read 6 Done Done Done false
52852023-12-10T18:51:51.815Z 6 Acked 1005 Read 8 Done Done Done false
52862023-12-10T18:51:51.815Z 7 Acked 1006 Read 5 Done Done Done false
52872023-12-10T18:51:51.815Z 8 Acked 1007 Write 6 Done Done Done false
52882023-12-10T18:51:51.815Z 9 Acked 1008 Read 1 Done Done Sent false
52892023-12-10T18:51:51.815Z 10 Acked 1009 Flush 0 Done Done Sent false
52902023-12-10T18:51:51.815Z 11 Acked 1010 Write 6 Done Done Sent false
52912023-12-10T18:51:51.815Z 12 Acked 1011 Read 2 Done Done Sent false
52922023-12-10T18:51:51.815Z 13 Acked 1012 Read 4 Done Done Sent false
52932023-12-10T18:51:51.815Z 14 Acked 1013 Write 5 Done Done Sent false
52942023-12-10T18:51:51.815Z 15 Acked 1014 Flush 0 Done Done Sent false
52952023-12-10T18:51:51.815Z 16 Acked 1015 Write 7 Done Done Sent false
52962023-12-10T18:51:51.816Z 17 Acked 1016 Read 1 Done Done Sent false
52972023-12-10T18:51:51.816Z 18 Acked 1017 Read 10 Done Done Sent false
52982023-12-10T18:51:51.816Z 19 Acked 1018 Read 4 Done Done Sent false
52992023-12-10T18:51:51.816Z 20 Acked 1019 Flush 0 Done Done Sent false
53002023-12-10T18:51:51.817Z 21 Acked 1020 Write 3 Done Done Sent false
53012023-12-10T18:51:51.817Z 22 Acked 1021 Write 8 Done Done Sent false
53022023-12-10T18:51:51.817Z 23 Acked 1022 Write 5 Done Done Sent false
53032023-12-10T18:51:51.817Z 24 Acked 1023 Read 1 Done Done Sent false
53042023-12-10T18:51:51.817Z 25 Acked 1024 Flush 0 Done Done Sent false
53052023-12-10T18:51:51.817Z 26 Acked 1025 Write 4 Done Done Sent false
53062023-12-10T18:51:51.817Z 27 Acked 1026 Read 2 Done Done Sent false
53072023-12-10T18:51:51.817Z 28 Acked 1027 Read 7 Done Done Sent false
53082023-12-10T18:51:51.817Z 29 Acked 1028 Write 3 Done Done Sent false
53092023-12-10T18:51:51.817Z 30 Acked 1029 Read 3 Done Sent Sent false
53102023-12-10T18:51:51.817Z STATES DS:0 DS:1 DS:2 TOTAL
53112023-12-10T18:51:51.817Z New 0 0 0 0
53122023-12-10T18:51:51.817Z Sent 0 1 22 23
53132023-12-10T18:51:51.817Z Done 30 29 8 67
53142023-12-10T18:51:51.817Z Skipped 0 0 0 0
53152023-12-10T18:51:51.818Z Error 0 0 0 0
53162023-12-10T18:51:51.818ZLast Flush: 1024 1024 0
53172023-12-10T18:51:51.818ZDownstairs last five completed:
53182023-12-10T18:51:51.818ZUpstairs last five completed: 30 29 28 27 26 25
53192023-12-10T18:51:51.818ZWrote out file "/tmp/test_repair_verify.data"
53202023-12-10T18:51:51.818Z
53212023-12-10T18:51:51.827ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
53222023-12-10T18:51:51.827ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53232023-12-10T18:51:51.827Z 0 000-019 41 41 41 68 68 68 F F F
53242023-12-10T18:51:51.827Z 1 020-039 42 42 32 71 71 54 F F F <---
53252023-12-10T18:51:51.827Z 2 040-059 42 42 37 71 71 59 F F F <---
53262023-12-10T18:51:51.827Z 3 060-079 38 38 38 62 62 62 F F F
53272023-12-10T18:51:51.827Z 4 080-099 36 36 36 57 57 57 F F F
53282023-12-10T18:51:51.827Z 5 100-119 36 36 36 57 57 57 F F F
53292023-12-10T18:51:51.827Z 6 120-139 31 31 31 53 53 53 F F F
53302023-12-10T18:51:51.827Z 7 140-159 38 38 38 62 62 62 F F F
53312023-12-10T18:51:51.828Z 8 160-179 42 42 37 69 69 59 T T T <---
53322023-12-10T18:51:51.828Z 9 180-199 42 42 41 69 69 68 T T T <---
53332023-12-10T18:51:51.828Z 10 200-219 41 41 41 68 68 68 F F F
53342023-12-10T18:51:51.828Z 11 220-239 42 42 41 72 72 68 F F F <---
53352023-12-10T18:51:51.828Z 12 240-259 42 42 41 72 72 68 F F F <---
53362023-12-10T18:51:51.828Z 13 260-279 37 37 37 59 59 59 F F F
53372023-12-10T18:51:51.828Z 14 280-299 38 38 38 60 60 60 F F F
53382023-12-10T18:51:51.828Z 15 300-319 42 42 41 70 70 68 F F F <---
53392023-12-10T18:51:51.828Z 16 320-339 42 42 41 72 72 68 F F F <---
53402023-12-10T18:51:51.829Z 17 340-359 37 37 37 59 59 59 F F F
53412023-12-10T18:51:51.829Z 18 360-379 42 42 41 72 72 68 F F F <---
53422023-12-10T18:51:51.829Z 19 380-399 41 41 41 68 68 68 F F F
53432023-12-10T18:51:51.829Z 20 400-419 42 42 38 70 70 64 F F F <---
53442023-12-10T18:51:51.829Z 21 420-439 33 33 33 55 55 55 F F F
53452023-12-10T18:51:51.829Z 22 440-459 37 37 37 59 59 59 F F F
53462023-12-10T18:51:51.829Z 23 460-479 40 40 40 67 67 67 F F F
53472023-12-10T18:51:51.829Z 24 480-499 36 36 36 58 58 58 F F F
53482023-12-10T18:51:51.829Z 25 500-519 38 38 38 63 63 63 F F F
53492023-12-10T18:51:51.829Z 26 520-539 42 42 38 69 69 63 F F T <---
53502023-12-10T18:51:51.830Z 27 540-559 42 42 40 69 69 66 F F T <---
53512023-12-10T18:51:51.830Z 28 560-579 40 40 40 66 66 66 F F F
53522023-12-10T18:51:51.830Z 29 580-599 38 38 38 61 61 61 F F F
53532023-12-10T18:51:51.830ZMax gen: 42, Max flush: 72
53542023-12-10T18:51:51.830ZError: Difference in extent metadata found!
53552023-12-10T18:51:51.830ZA Difference in extent metadata is expected here
53562023-12-10T18:51:51.830ZOn loop 21
53572023-12-10T18:51:51.830Z
53582023-12-10T18:51:51.830ZVerifying data now
53592023-12-10T18:51:57.975ZNo change after verify
53602023-12-10T18:51:57.975ZLoop: 21 Downstairs dump after verify (and repair):
53612023-12-10T18:51:57.999ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
53622023-12-10T18:51:57.999Z 0 000-019 41 41 41 68 68 68 F F F
53632023-12-10T18:51:57.999Z 1 020-039 42 42 42 71 71 71 F F F
53642023-12-10T18:51:57.999Z 2 040-059 42 42 42 71 71 71 F F F
53652023-12-10T18:51:57.999Z 3 060-079 38 38 38 62 62 62 F F F
53662023-12-10T18:51:57.999Z 4 080-099 36 36 36 57 57 57 F F F
53672023-12-10T18:51:57.999Z 5 100-119 36 36 36 57 57 57 F F F
53682023-12-10T18:51:58.003Z 6 120-139 31 31 31 53 53 53 F F F
53692023-12-10T18:51:58.003Z 7 140-159 38 38 38 62 62 62 F F F
53702023-12-10T18:51:58.004Z 8 160-179 43 43 43 73 73 73 F F F
53712023-12-10T18:51:58.004Z 9 180-199 43 43 43 73 73 73 F F F
53722023-12-10T18:51:58.004Z 10 200-219 41 41 41 68 68 68 F F F
53732023-12-10T18:51:58.004Z 11 220-239 42 42 42 72 72 72 F F F
53742023-12-10T18:51:58.004Z 12 240-259 42 42 42 72 72 72 F F F
53752023-12-10T18:51:58.004Z 13 260-279 37 37 37 59 59 59 F F F
53762023-12-10T18:51:58.004Z 14 280-299 38 38 38 60 60 60 F F F
53772023-12-10T18:51:58.004Z 15 300-319 42 42 42 70 70 70 F F F
53782023-12-10T18:51:58.004Z 16 320-339 42 42 42 72 72 72 F F F
53792023-12-10T18:51:58.004Z 17 340-359 37 37 37 59 59 59 F F F
53802023-12-10T18:51:58.004Z 18 360-379 42 42 42 72 72 72 F F F
53812023-12-10T18:51:58.004Z 19 380-399 41 41 41 68 68 68 F F F
53822023-12-10T18:51:58.005Z 20 400-419 42 42 42 70 70 70 F F F
53832023-12-10T18:51:58.005Z 21 420-439 33 33 33 55 55 55 F F F
53842023-12-10T18:51:58.005Z 22 440-459 37 37 37 59 59 59 F F F
53852023-12-10T18:51:58.005Z 23 460-479 40 40 40 67 67 67 F F F
53862023-12-10T18:51:58.005Z 24 480-499 36 36 36 58 58 58 F F F
53872023-12-10T18:51:58.005Z 25 500-519 38 38 38 63 63 63 F F F
53882023-12-10T18:51:58.005Z 26 520-539 42 42 42 69 69 69 F F F
53892023-12-10T18:51:58.005Z 27 540-559 42 42 42 69 69 69 F F F
53902023-12-10T18:51:58.005Z 28 560-579 40 40 40 66 66 66 F F F
53912023-12-10T18:51:58.005Z 29 580-599 38 38 38 61 61 61 F F F
53922023-12-10T18:51:58.005ZMax gen: 43, Max flush: 73
53932023-12-10T18:51:58.008Z
53942023-12-10T18:51:58.009ZBegin loop 22
53952023-12-10T18:51:58.010ZDownstairs to restart: 1
53962023-12-10T18:51:58.038Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.037548837Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338}
53972023-12-10T18:51:58.038Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.037972132Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338}
53982023-12-10T18:51:58.038Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.038001119Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338}
53992023-12-10T18:51:58.043Z{"msg":"Crucible 223a7aaf-c6bd-4e98-a192-2419f684a763 has session id: 6d8bc022-48a6-45c7-81cc-045593e55a6c","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.038061178Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54002023-12-10T18:51:58.043Z{"msg":"Upstairs opts: Upstairs UUID: 223a7aaf-c6bd-4e98-a192-2419f684a763, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.038154682Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54012023-12-10T18:51:58.045Z{"msg":"Crucible stats registered with UUID: 223a7aaf-c6bd-4e98-a192-2419f684a763","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.03819705Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338Crucible runtime is spawned
54022023-12-10T18:51:58.045Z,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54032023-12-10T18:51:58.045ZThe guest has requested activation with gen:44
54042023-12-10T18:51:58.045Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.038502921Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"looper":"2","session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54052023-12-10T18:51:58.045Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.038658999Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c","task":"up_listen"}
54062023-12-10T18:51:58.045Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.038706895Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54072023-12-10T18:51:58.045Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.038803443Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54082023-12-10T18:51:58.045Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.038847834Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"looper":"0","session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54092023-12-10T18:51:58.045Z{"msg":"[2] 223a7aaf-c6bd-4e98-a192-2419f684a763 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.038950995Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"looper":"2","session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54102023-12-10T18:51:58.045Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.039062993Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"looper":"1","session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54112023-12-10T18:51:58.045Z{"msg":"[0] 223a7aaf-c6bd-4e98-a192-2419f684a763 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.039195037Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"looper":"0","session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54122023-12-10T18:51:58.045Z{"msg":"[1] 223a7aaf-c6bd-4e98-a192-2419f684a763 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.039352417Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"looper":"1","session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54132023-12-10T18:51:58.045Z{"msg":"223a7aaf-c6bd-4e98-a192-2419f684a763 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.039602179Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54142023-12-10T18:51:58.045Z{"msg":"Set desired generation to :44","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.039655916Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54152023-12-10T18:51:58.046Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.039751355Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54162023-12-10T18:51:58.046Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.039988048Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54172023-12-10T18:51:58.046Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.040178429Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54182023-12-10T18:51:58.046Z{"msg":"[0] 223a7aaf-c6bd-4e98-a192-2419f684a763 (6d8bc022-48a6-45c7-81cc-045593e55a6c) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.040564678Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54192023-12-10T18:51:58.046Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.04063236Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54202023-12-10T18:51:58.046Z{"msg":"[2] 223a7aaf-c6bd-4e98-a192-2419f684a763 (6d8bc022-48a6-45c7-81cc-045593e55a6c) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.040660241Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54212023-12-10T18:51:58.046Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.040708892Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54222023-12-10T18:51:58.046Z{"msg":"[0] client is_active_req TRUE, promote! session 6d8bc022-48a6-45c7-81cc-045593e55a6c","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.040736146Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54232023-12-10T18:51:58.047Z{"msg":"[2] client is_active_req TRUE, promote! session 6d8bc022-48a6-45c7-81cc-045593e55a6c","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.04075811Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54242023-12-10T18:51:58.047Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.041582236Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54252023-12-10T18:51:58.047Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.041664847Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54262023-12-10T18:51:58.047Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.041717162Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54272023-12-10T18:51:58.047Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.041783515Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54282023-12-10T18:51:58.047Z{"msg":"223a7aaf-c6bd-4e98-a192-2419f684a763 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.041815609Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54292023-12-10T18:51:58.047Z{"msg":"223a7aaf-c6bd-4e98-a192-2419f684a763 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.041884153Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54302023-12-10T18:51:58.053Z{"msg":"[1] 223a7aaf-c6bd-4e98-a192-2419f684a763 (6d8bc022-48a6-45c7-81cc-045593e55a6c) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.042113306Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54312023-12-10T18:51:58.053Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.042352906Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54322023-12-10T18:51:58.053Z{"msg":"[1] client is_active_req TRUE, promote! session 6d8bc022-48a6-45c7-81cc-045593e55a6c","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.04258275Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54332023-12-10T18:51:58.053Z{"msg":"[2] 223a7aaf-c6bd-4e98-a192-2419f684a763 (6d8bc022-48a6-45c7-81cc-045593e55a6c) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.042630823Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54342023-12-10T18:51:58.053Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.042664424Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54352023-12-10T18:51:58.053Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:58.042696635Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54362023-12-10T18:51:58.053Z{"msg":"[0] 223a7aaf-c6bd-4e98-a192-2419f684a763 (6d8bc022-48a6-45c7-81cc-045593e55a6c) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.042747697Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54372023-12-10T18:51:58.053Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.042820429Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54382023-12-10T18:51:58.054Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:58.04289211Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54392023-12-10T18:51:58.054Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.042965183Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54402023-12-10T18:51:58.054Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.043046258Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54412023-12-10T18:51:58.054Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.043089187Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54422023-12-10T18:51:58.054Z{"msg":"223a7aaf-c6bd-4e98-a192-2419f684a763 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.043147135Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54432023-12-10T18:51:58.054Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.043170073Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54442023-12-10T18:51:58.054Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.043189983Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54452023-12-10T18:51:58.054Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.043247961Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54462023-12-10T18:51:58.054Z{"msg":"223a7aaf-c6bd-4e98-a192-2419f684a763 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.043271008Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54472023-12-10T18:51:58.054Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.043291882Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54482023-12-10T18:51:58.054Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.043353821Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54492023-12-10T18:51:58.054Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.043429371Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54502023-12-10T18:51:58.054Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.043479252Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54512023-12-10T18:51:58.054Z{"msg":"223a7aaf-c6bd-4e98-a192-2419f684a763 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.04354573Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54522023-12-10T18:51:58.054Z{"msg":"[1] 223a7aaf-c6bd-4e98-a192-2419f684a763 (6d8bc022-48a6-45c7-81cc-045593e55a6c) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.044189186Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54532023-12-10T18:51:58.054Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.044249617Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54542023-12-10T18:51:58.054Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:51:58.044274496Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54552023-12-10T18:51:58.054Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.044294698Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54562023-12-10T18:51:58.055Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.044314921Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54572023-12-10T18:51:58.055Z{"msg":"223a7aaf-c6bd-4e98-a192-2419f684a763 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.044356093Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54582023-12-10T18:51:58.055Z{"msg":"[0]R flush_numbers[0..12]: [68, 71, 71, 62, 57, 57, 53, 62, 73, 73, 68, 72]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.044379669Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54592023-12-10T18:51:58.055Z{"msg":"[0]R generation[0..12]: [41, 42, 42, 38, 36, 36, 31, 38, 43, 43, 41, 42]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.044400905Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54602023-12-10T18:51:58.055Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.044422881Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54612023-12-10T18:51:58.055Z{"msg":"[1]R flush_numbers[0..12]: [68, 71, 71, 62, 57, 57, 53, 62, 73, 73, 68, 72]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:44
54622023-12-10T18:51:58.055ZWait for a query_work_queue command to finish before sending IO
54632023-12-10T18:51:58.055Z,"time":"2023-12-10T18:51:58.044443816Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54642023-12-10T18:51:58.055Z{"msg":"[1]R generation[0..12]: [41, 42, 42, 38, 36, 36, 31, 38, 43, 43, 41, 42]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.044682113Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54652023-12-10T18:51:58.055Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.044730566Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54662023-12-10T18:51:58.055Z{"msg":"[2]R flush_numbers[0..12]: [68, 71, 71, 62, 57, 57, 53, 62, 73, 73, 68, 72]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.044771328Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54672023-12-10T18:51:58.055Z{"msg":"[2]R generation[0..12]: [41, 42, 42, 38, 36, 36, 31, 38, 43, 43, 41, 42]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.044807965Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54682023-12-10T18:51:58.055Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.044881178Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54692023-12-10T18:51:58.055Z{"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.044949786Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54702023-12-10T18:51:58.055Z{"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.045016758Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54712023-12-10T18:51:58.055Z{"msg":"Next flush: 74","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.045040758Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54722023-12-10T18:51:58.056Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.045080861Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54732023-12-10T18:51:58.056Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.04510304Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54742023-12-10T18:51:58.056Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.045165119Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54752023-12-10T18:51:58.056Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.04523299Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54762023-12-10T18:51:58.056Z{"msg":"223a7aaf-c6bd-4e98-a192-2419f684a763 is now active with session: 6d8bc022-48a6-45c7-81cc-045593e55a6c","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.045277395Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54772023-12-10T18:51:58.056Z{"msg":"223a7aaf-c6bd-4e98-a192-2419f684a763 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.045315345Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54782023-12-10T18:51:58.056Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.045355394Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54792023-12-10T18:51:58.057Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
54802023-12-10T18:51:58.057Z,"time":"2023-12-10T18:51:58.045422064Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54812023-12-10T18:51:58.057Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.045467289Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54822023-12-10T18:51:58.057Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.045516146Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54832023-12-10T18:51:58.058Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.045539227Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54842023-12-10T18:51:58.058Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.045579327Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54852023-12-10T18:51:58.058Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.045600426Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54862023-12-10T18:51:58.058Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.045620314Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54872023-12-10T18:51:58.058Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.045661647Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54882023-12-10T18:51:58.058Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.045684243Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54892023-12-10T18:51:58.058Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:51:58.045703817Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1338,"session_id":"6d8bc022-48a6-45c7-81cc-045593e55a6c"}
54902023-12-10T18:51:58.058ZLoading write count information from file "/tmp/test_repair_verify.data"
54912023-12-10T18:51:58.058ZRun Repair workload
54922023-12-10T18:51:58.058Z01/30 Write block 49 len 32768 data: 5 5 5 4 3 3 3 3
54932023-12-10T18:51:58.058Z02/30 Read block 303 len 4096
54942023-12-10T18:51:58.058Z03/30 Write block 458 len 4096 data: 5
54952023-12-10T18:51:58.058Z04/30 Read block 588 len 8192
54962023-12-10T18:51:58.059Z05/30 Read block 347 len 20480
54972023-12-10T18:51:58.062Z06/30 Write block 521 len 4096 data: 4
54982023-12-10T18:51:58.062Z07/30 Read block 11 len 12288
54992023-12-10T18:51:58.063Z08/30 Read block 467 len 24576
55002023-12-10T18:51:58.063Z09/30 Read block 343 len 36864
55012023-12-10T18:51:58.063Z10/30 Read block 535 len 12288
55022023-12-10T18:51:58.063Z11/30 Write block 410 len 4096 data: 5
55032023-12-10T18:51:58.066Z12/30 Read block 365 len 36864
55042023-12-10T18:51:58.066Z13/30 Read block 152 len 16384
55052023-12-10T18:51:58.069Z14/30 Read block 155 len 32768
55062023-12-10T18:51:58.071Z15/30 Read block 393 len 16384
55072023-12-10T18:51:58.074Z16/30 Read block 265 len 40960
55082023-12-10T18:51:58.080Z17/30 Write block 245 len 36864 data: 8 7 6 4 6 5 5 4 5
55092023-12-10T18:51:58.080Z18/30 Write block 120 len 4096 data: 8
55102023-12-10T18:51:58.080Z19/30 Write block 472 len 8192 data: 7 6
55112023-12-10T18:51:58.083Z20/30 Read block 224 len 16384
55122023-12-10T18:51:58.083Z21/30 Read block 591 len 8192
55132023-12-10T18:51:58.083Z22/30 Read block 179 len 16384
55142023-12-10T18:51:58.083Z23/30 Read block 258 len 40960
55152023-12-10T18:51:58.086Z24/30 Write block 127 len 20480 data: 5 5 5 3 3
55162023-12-10T18:51:58.089Z25/30 Flush
55172023-12-10T18:51:58.092Z26/30 Write block 488 len 16384 data: 5 5 5 6
55182023-12-10T18:51:58.092Z27/30 Write block 96 len 20480 data: 2 2 3 4 4
55192023-12-10T18:51:58.095Z28/30 Read block 172 len 36864
55202023-12-10T18:51:58.098Z29/30 Read block 390 len 4096
55212023-12-10T18:51:58.098Z30/30 Write block 342 len 36864 data: 3 4 5 5 4 6 4 5 4
55222023-12-10T18:51:58.101Z----------------------------------------------------------------
55232023-12-10T18:51:58.101Z Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30
55242023-12-10T18:51:58.101ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
55252023-12-10T18:51:58.101Z 1 Acked 1000 Write 8 Done Sent Done false
55262023-12-10T18:51:58.101Z 2 Acked 1001 Read 1 Done Sent Done false
55272023-12-10T18:51:58.101Z 3 Acked 1002 Write 1 Done Sent Done false
55282023-12-10T18:51:58.101Z 4 Acked 1003 Read 2 Done Sent Done false
55292023-12-10T18:51:58.101Z 5 Acked 1004 Read 5 Done Sent Done false
55302023-12-10T18:51:58.101Z 6 Acked 1005 Write 1 Done Sent Done false
55312023-12-10T18:51:58.101Z 7 Acked 1006 Read 3 Done Sent Done false
55322023-12-10T18:51:58.101Z 8 Acked 1007 Read 6 Done Sent Done false
55332023-12-10T18:51:58.102Z 9 Acked 1008 Read 9 Done Sent Done false
55342023-12-10T18:51:58.102Z 10 Acked 1009 Read 3 Done Sent Done false
55352023-12-10T18:51:58.102Z 11 Acked 1010 Write 1 Done Sent Done false
55362023-12-10T18:51:58.102Z 12 Acked 1011 Read 9 Done Sent Done false
55372023-12-10T18:51:58.102Z 13 Acked 1012 Read 4 Done Sent Done false
55382023-12-10T18:51:58.102Z 14 Acked 1013 Read 8 Done Sent Done false
55392023-12-10T18:51:58.102Z 15 Acked 1014 Read 4 Done Sent Done false
55402023-12-10T18:51:58.102Z 16 Acked 1015 Read 10 Done Sent Done false
55412023-12-10T18:51:58.102Z 17 Acked 1016 Write 9 Done Sent Done false
55422023-12-10T18:51:58.102Z 18 Acked 1017 Write 1 Done Sent Done false
55432023-12-10T18:51:58.102Z 19 Acked 1018 Write 2 Done Sent Done false
55442023-12-10T18:51:58.102Z 20 Acked 1019 Read 4 Done Sent Done false
55452023-12-10T18:51:58.102Z 21 Acked 1020 Read 2 Done Sent Done false
55462023-12-10T18:51:58.102Z 22 Acked 1021 Read 4 Done Sent Done false
55472023-12-10T18:51:58.102Z 23 Acked 1022 Read 10 Done Sent Done false
55482023-12-10T18:51:58.103Z 24 Acked 1023 Write 5 Done Sent Done false
55492023-12-10T18:51:58.103Z 25 Acked 1024 Flush 0 Done Sent Done false
55502023-12-10T18:51:58.103Z 26 Acked 1025 Write 4 Done Sent Done false
55512023-12-10T18:51:58.103Z 27 Acked 1026 Write 5 Done Sent Done false
55522023-12-10T18:51:58.103Z 28 Acked 1027 Read 9 Done Sent Done false
55532023-12-10T18:51:58.103Z 29 Acked 1028 Read 1 Done Sent Done false
55542023-12-10T18:51:58.103Z 30 Acked 1029 Write 9 Sent Sent Sent false
55552023-12-10T18:51:58.103Z STATES DS:0 DS:1 DS:2 TOTAL
55562023-12-10T18:51:58.104Z New 0 0 0 0
55572023-12-10T18:51:58.104Z Sent 1 30 1 32
55582023-12-10T18:51:58.104Z Done 29 0 29 58
55592023-12-10T18:51:58.104Z Skipped 0 0 0 0
55602023-12-10T18:51:58.104Z Error 0 0 0 0
55612023-12-10T18:51:58.104ZLast Flush: 1024 0 1024
55622023-12-10T18:51:58.104ZDownstairs last five completed:
55632023-12-10T18:51:58.104ZUpstairs last five completed: 30 29 28 27 26 25
55642023-12-10T18:51:58.104ZWrote out file "/tmp/test_repair_verify.data"
55652023-12-10T18:51:58.107Z
55662023-12-10T18:51:58.114ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
55672023-12-10T18:51:58.138ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
55682023-12-10T18:51:58.138Z 0 000-019 41 41 41 68 68 68 F F F
55692023-12-10T18:51:58.138Z 1 020-039 42 42 42 71 71 71 F F F
55702023-12-10T18:51:58.138Z 2 040-059 44 42 44 74 71 74 F F F <---
55712023-12-10T18:51:58.145Z 3 060-079 38 38 38 62 62 62 F F F
55722023-12-10T18:51:58.145Z 4 080-099 36 36 36 57 57 57 T F T <---
55732023-12-10T18:51:58.145Z 5 100-119 36 36 36 57 57 57 T F T <---
55742023-12-10T18:51:58.149Z 6 120-139 44 31 44 74 53 74 F F F <---
55752023-12-10T18:51:58.149Z 7 140-159 38 38 38 62 62 62 F F F
55762023-12-10T18:51:58.149Z 8 160-179 43 43 43 73 73 73 F F F
55772023-12-10T18:51:58.149Z 9 180-199 43 43 43 73 73 73 F F F
55782023-12-10T18:51:58.149Z 10 200-219 41 41 41 68 68 68 F F F
55792023-12-10T18:51:58.149Z 11 220-239 42 42 42 72 72 72 F F F
55802023-12-10T18:51:58.150Z 12 240-259 44 42 44 74 72 74 F F F <---
55812023-12-10T18:51:58.150Z 13 260-279 37 37 37 59 59 59 F F F
55822023-12-10T18:51:58.150Z 14 280-299 38 38 38 60 60 60 F F F
55832023-12-10T18:51:58.150Z 15 300-319 42 42 42 70 70 70 F F F
55842023-12-10T18:51:58.150Z 16 320-339 42 42 42 72 72 72 F F F
55852023-12-10T18:51:58.150Z 17 340-359 37 37 37 59 59 59 T F T <---
55862023-12-10T18:51:58.150Z 18 360-379 42 42 42 72 72 72 F F F
55872023-12-10T18:51:58.150Z 19 380-399 41 41 41 68 68 68 F F F
55882023-12-10T18:51:58.150Z 20 400-419 44 42 44 74 70 74 F F F <---
55892023-12-10T18:51:58.150Z 21 420-439 33 33 33 55 55 55 F F F
55902023-12-10T18:51:58.150Z 22 440-459 44 37 44 74 59 74 F F F <---
55912023-12-10T18:51:58.150Z 23 460-479 44 40 44 74 67 74 F F F <---
55922023-12-10T18:51:58.151Z 24 480-499 36 36 36 58 58 58 T F T <---
55932023-12-10T18:51:58.152Z 25 500-519 38 38 38 63 63 63 F F F
55942023-12-10T18:51:58.152Z 26 520-539 44 42 44 74 69 74 F F F <---
55952023-12-10T18:51:58.152Z 27 540-559 42 42 42 69 69 69 F F F
55962023-12-10T18:51:58.152Z 28 560-579 40 40 40 66 66 66 F F F
55972023-12-10T18:51:58.152Z 29 580-599 38 38 38 61 61 61 F F F
55982023-12-10T18:51:58.152ZMax gen: 44, Max flush: 74
55992023-12-10T18:51:58.152ZError: Difference in extent metadata found!
56002023-12-10T18:51:58.152ZA Difference in extent metadata is expected here
56012023-12-10T18:51:58.152ZOn loop 22
56022023-12-10T18:51:58.152Z
56032023-12-10T18:51:58.152ZVerifying data now
56042023-12-10T18:52:03.267ZNo change after verify
56052023-12-10T18:52:03.267ZLoop: 22 Downstairs dump after verify (and repair):
56062023-12-10T18:52:03.291ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
56072023-12-10T18:52:03.291Z 0 000-019 41 41 41 68 68 68 F F F
56082023-12-10T18:52:03.291Z 1 020-039 42 42 42 71 71 71 F F F
56092023-12-10T18:52:03.291Z 2 040-059 44 44 44 74 74 74 F F F
56102023-12-10T18:52:03.292Z 3 060-079 38 38 38 62 62 62 F F F
56112023-12-10T18:52:03.292Z 4 080-099 45 45 45 75 75 75 F F F
56122023-12-10T18:52:03.292Z 5 100-119 45 45 45 75 75 75 F F F
56132023-12-10T18:52:03.292Z 6 120-139 44 44 44 74 74 74 F F F
56142023-12-10T18:52:03.295Z 7 140-159 38 38 38 62 62 62 F F F
56152023-12-10T18:52:03.295Z 8 160-179 43 43 43 73 73 73 F F F
56162023-12-10T18:52:03.296Z 9 180-199 43 43 43 73 73 73 F F F
56172023-12-10T18:52:03.296Z 10 200-219 41 41 41 68 68 68 F F F
56182023-12-10T18:52:03.296Z 11 220-239 42 42 42 72 72 72 F F F
56192023-12-10T18:52:03.296Z 12 240-259 44 44 44 74 74 74 F F F
56202023-12-10T18:52:03.296Z 13 260-279 37 37 37 59 59 59 F F F
56212023-12-10T18:52:03.296Z 14 280-299 38 38 38 60 60 60 F F F
56222023-12-10T18:52:03.296Z 15 300-319 42 42 42 70 70 70 F F F
56232023-12-10T18:52:03.296Z 16 320-339 42 42 42 72 72 72 F F F
56242023-12-10T18:52:03.296Z 17 340-359 45 45 45 75 75 75 F F F
56252023-12-10T18:52:03.296Z 18 360-379 42 42 42 72 72 72 F F F
56262023-12-10T18:52:03.296Z 19 380-399 41 41 41 68 68 68 F F F
56272023-12-10T18:52:03.296Z 20 400-419 44 44 44 74 74 74 F F F
56282023-12-10T18:52:03.296Z 21 420-439 33 33 33 55 55 55 F F F
56292023-12-10T18:52:03.296Z 22 440-459 44 44 44 74 74 74 F F F
56302023-12-10T18:52:03.296Z 23 460-479 44 44 44 74 74 74 F F F
56312023-12-10T18:52:03.296Z 24 480-499 45 45 45 75 75 75 F F F
56322023-12-10T18:52:03.296Z 25 500-519 38 38 38 63 63 63 F F F
56332023-12-10T18:52:03.296Z 26 520-539 44 44 44 74 74 74 F F F
56342023-12-10T18:52:03.297Z 27 540-559 42 42 42 69 69 69 F F F
56352023-12-10T18:52:03.297Z 28 560-579 40 40 40 66 66 66 F F F
56362023-12-10T18:52:03.297Z 29 580-599 38 38 38 61 61 61 F F F
56372023-12-10T18:52:03.297ZMax gen: 45, Max flush: 75
56382023-12-10T18:52:03.301Z
56392023-12-10T18:52:03.301ZBegin loop 23
56402023-12-10T18:52:03.301ZDownstairs to restart: 0
56412023-12-10T18:52:03.329Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.328950449Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346}
56422023-12-10T18:52:03.329Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.329349833Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346}
56432023-12-10T18:52:03.329Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.329377909Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346}
56442023-12-10T18:52:03.329Z{"msg":"Crucible 4f04a0ec-5d4f-4dce-b2d2-5d08a1534b02 has session id: 9f09b045-31cd-41e1-828f-370c5da30386","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.329399246Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56452023-12-10T18:52:03.345Z{"msg":"Upstairs opts: Upstairs UUID: 4f04a0ec-5d4f-4dce-b2d2-5d08a1534b02, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.329472866Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56462023-12-10T18:52:03.345Z{"msg":"Crucible stats registered with UUID: 4f04a0ec-5d4f-4dce-b2d2-5d08a1534b02","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.329544275Z","hostname":"Crucible runtime is spawned
56472023-12-10T18:52:03.345Zip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56482023-12-10T18:52:03.345ZThe guest has requested activation with gen:46
56492023-12-10T18:52:03.345Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.329749501Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"looper":"0","session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56502023-12-10T18:52:03.345Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.329861718Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"looper":"1","session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56512023-12-10T18:52:03.345Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.329903872Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"looper":"2","session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56522023-12-10T18:52:03.345Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.329943312Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386","task":"up_listen"}
56532023-12-10T18:52:03.345Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.329980049Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56542023-12-10T18:52:03.345Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.330077733Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56552023-12-10T18:52:03.345Z{"msg":"4f04a0ec-5d4f-4dce-b2d2-5d08a1534b02 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.330122117Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56562023-12-10T18:52:03.345Z{"msg":"Set desired generation to :46","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.330165051Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56572023-12-10T18:52:03.345Z{"msg":"[2] 4f04a0ec-5d4f-4dce-b2d2-5d08a1534b02 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.330306468Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"looper":"2","session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56582023-12-10T18:52:03.345Z{"msg":"[1] 4f04a0ec-5d4f-4dce-b2d2-5d08a1534b02 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.330370623Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"looper":"1","session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56592023-12-10T18:52:03.345Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.330397154Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56602023-12-10T18:52:03.346Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.330723875Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56612023-12-10T18:52:03.347Z{"msg":"[2] 4f04a0ec-5d4f-4dce-b2d2-5d08a1534b02 (9f09b045-31cd-41e1-828f-370c5da30386) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.331118842Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56622023-12-10T18:52:03.347Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.331185319Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56632023-12-10T18:52:03.347Z{"msg":"[2] client is_active_req TRUE, promote! session 9f09b045-31cd-41e1-828f-370c5da30386","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.33121126Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56642023-12-10T18:52:03.347Z{"msg":"[1] 4f04a0ec-5d4f-4dce-b2d2-5d08a1534b02 (9f09b045-31cd-41e1-828f-370c5da30386) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.331265312Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56652023-12-10T18:52:03.347Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.331315515Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56662023-12-10T18:52:03.347Z{"msg":"[1] client is_active_req TRUE, promote! session 9f09b045-31cd-41e1-828f-370c5da30386","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.33149214Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56672023-12-10T18:52:03.347Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.33190943Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56682023-12-10T18:52:03.347Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.332020058Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56692023-12-10T18:52:03.348Z{"msg":"4f04a0ec-5d4f-4dce-b2d2-5d08a1534b02 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.332063862Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56702023-12-10T18:52:03.348Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.332094213Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56712023-12-10T18:52:03.348Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.332138889Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56722023-12-10T18:52:03.348Z{"msg":"4f04a0ec-5d4f-4dce-b2d2-5d08a1534b02 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.332177924Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56732023-12-10T18:52:03.348Z{"msg":"[2] 4f04a0ec-5d4f-4dce-b2d2-5d08a1534b02 (9f09b045-31cd-41e1-828f-370c5da30386) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.332728836Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56742023-12-10T18:52:03.348Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.332773193Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56752023-12-10T18:52:03.348Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:03.332813345Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56762023-12-10T18:52:03.348Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.332851124Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56772023-12-10T18:52:03.348Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.332918572Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56782023-12-10T18:52:03.348Z{"msg":"4f04a0ec-5d4f-4dce-b2d2-5d08a1534b02 New WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.333010138Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56792023-12-10T18:52:03.348Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.33305329Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56802023-12-10T18:52:03.348Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.333113756Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56812023-12-10T18:52:03.349Z{"msg":"[1] 4f04a0ec-5d4f-4dce-b2d2-5d08a1534b02 (9f09b045-31cd-41e1-828f-370c5da30386) New WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.333158439Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56822023-12-10T18:52:03.349Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.3332244Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56832023-12-10T18:52:03.349Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:03.333265039Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56842023-12-10T18:52:03.349Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.333322291Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56852023-12-10T18:52:03.349Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.333363322Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56862023-12-10T18:52:03.349Z{"msg":"4f04a0ec-5d4f-4dce-b2d2-5d08a1534b02 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.333420682Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56872023-12-10T18:52:03.349Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.333460825Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56882023-12-10T18:52:03.349Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:03.333515501Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56892023-12-10T18:52:04.330Z{"msg":"[0] 4f04a0ec-5d4f-4dce-b2d2-5d08a1534b02 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.331375268Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"looper":"0","session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56902023-12-10T18:52:04.330Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.331469976Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56912023-12-10T18:52:04.336Z{"msg":"[0] 4f04a0ec-5d4f-4dce-b2d2-5d08a1534b02 (9f09b045-31cd-41e1-828f-370c5da30386) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.332460485Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56922023-12-10T18:52:04.337Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.332518951Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56932023-12-10T18:52:04.337Z{"msg":"[0] client is_active_req TRUE, promote! session 9f09b045-31cd-41e1-828f-370c5da30386","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.332548182Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56942023-12-10T18:52:04.337Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.333194124Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56952023-12-10T18:52:04.337Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.33330411Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56962023-12-10T18:52:04.338Z{"msg":"4f04a0ec-5d4f-4dce-b2d2-5d08a1534b02 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.333372688Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56972023-12-10T18:52:04.338Z{"msg":"[0] 4f04a0ec-5d4f-4dce-b2d2-5d08a1534b02 (9f09b045-31cd-41e1-828f-370c5da30386) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.333953193Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56982023-12-10T18:52:04.338Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.334031248Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
56992023-12-10T18:52:04.338Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:04.33407108Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57002023-12-10T18:52:04.338Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.334125386Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57012023-12-10T18:52:04.338Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.334149784Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57022023-12-10T18:52:04.338Z{"msg":"4f04a0ec-5d4f-4dce-b2d2-5d08a1534b02 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.334207929Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57032023-12-10T18:52:04.338Z{"msg":"[0]R flush_numbers[0..12]: [68, 71, 74, 62, 75, 75, 74, 62, 73, 73, 68, 72]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.334240554Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57042023-12-10T18:52:04.338Z{"msg":"[0]R generation[0..12]: [41, 42, 44, 38, 45, 45, 44, 38, 43, 43, 41, 42]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:46
57052023-12-10T18:52:04.338Z,"time":"2023-12-10T18:52:04.334295749Z","Wait for a query_work_queue command to finish before sending IO
57062023-12-10T18:52:04.338Zhostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57072023-12-10T18:52:04.338Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.334332044Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57082023-12-10T18:52:04.338Z{"msg":"[1]R flush_numbers[0..12]: [68, 71, 74, 62, 75, 75, 74, 62, 73, 73, 68, 72]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.334387719Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57092023-12-10T18:52:04.338Z{"msg":"[1]R generation[0..12]: [41, 42, 44, 38, 45, 45, 44, 38, 43, 43, 41, 42]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.334412855Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57102023-12-10T18:52:04.339Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.334439323Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57112023-12-10T18:52:04.339Z{"msg":"[2]R flush_numbers[0..12]: [68, 71, 74, 62, 75, 75, 74, 62, 73, 73, 68, 72]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.334564876Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57122023-12-10T18:52:04.339Z{"msg":"[2]R generation[0..12]: [41, 42, 44, 38, 45, 45, 44, 38, 43, 43, 41, 42]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.334605224Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57132023-12-10T18:52:04.339Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.334643841Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57142023-12-10T18:52:04.339Z{"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.334704333Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57152023-12-10T18:52:04.339Z{"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.334745094Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57162023-12-10T18:52:04.339Z{"msg":"Next flush: 76","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.334780678Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57172023-12-10T18:52:04.339Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.334817086Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57182023-12-10T18:52:04.340Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.334871231Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57192023-12-10T18:52:04.340Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.334910565Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57202023-12-10T18:52:04.340Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.334976013Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57212023-12-10T18:52:04.340Z{"msg":"4f04a0ec-5d4f-4dce-b2d2-5d08a1534b02 is now active with session: 9f09b045-31cd-41e1-828f-370c5da30386","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.335017135Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57222023-12-10T18:52:04.340Z{"msg":"4f04a0ec-5d4f-4dce-b2d2-5d08a1534b02 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.335080488Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57232023-12-10T18:52:04.340Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
57242023-12-10T18:52:04.340Z2023-12-10T18:52:04.335120283Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57252023-12-10T18:52:04.340Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.335169492Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57262023-12-10T18:52:04.340Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.335206617Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57272023-12-10T18:52:04.344Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.335243726Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57282023-12-10T18:52:04.344Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.335298232Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57292023-12-10T18:52:04.344Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.335336707Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57302023-12-10T18:52:04.344Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.335392043Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57312023-12-10T18:52:04.345Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.335430187Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57322023-12-10T18:52:04.345Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.335485356Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57332023-12-10T18:52:04.345Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.335523259Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57342023-12-10T18:52:04.345Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:04.33558671Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1346,"session_id":"9f09b045-31cd-41e1-828f-370c5da30386"}
57352023-12-10T18:52:04.345ZLoading write count information from file "/tmp/test_repair_verify.data"
57362023-12-10T18:52:04.345ZRun Repair workload
57372023-12-10T18:52:04.345Z01/30 Write block 124 len 28672 data: 9 7 7 6 6 6 4
57382023-12-10T18:52:04.345Z02/30 Write block 219 len 32768 data: 5 5 7 7 6 6 5 5
57392023-12-10T18:52:04.345Z03/30 Write block 85 len 24576 data: 7 8 6 5 4 3
57402023-12-10T18:52:04.345Z04/30 Read block 240 len 16384
57412023-12-10T18:52:04.345Z05/30 Write block 215 len 12288 data: 8 9 8
57422023-12-10T18:52:04.345Z06/30 Read block 321 len 28672
57432023-12-10T18:52:04.349Z07/30 Flush
57442023-12-10T18:52:04.352Z08/30 Write block 84 len 32768 data: 7 8 9 7 6 5 4 3
57452023-12-10T18:52:04.352Z09/30 Read block 25 len 20480
57462023-12-10T18:52:04.359Z10/30 Read block 586 len 4096
57472023-12-10T18:52:04.362Z11/30 Read block 235 len 4096
57482023-12-10T18:52:04.362Z12/30 Read block 67 len 16384
57492023-12-10T18:52:04.362Z13/30 Read block 238 len 32768
57502023-12-10T18:52:04.362Z14/30 Read block 264 len 24576
57512023-12-10T18:52:04.362Z15/30 Write block 453 len 28672 data: 3 3 3 5 5 6 5
57522023-12-10T18:52:04.362Z16/30 Write block 4 len 16384 data: 3 3 3 3
57532023-12-10T18:52:04.367Z17/30 Read block 489 len 4096
57542023-12-10T18:52:04.367Z18/30 Read block 438 len 40960
57552023-12-10T18:52:04.371Z19/30 Read block 282 len 36864
57562023-12-10T18:52:04.378Z20/30 Read block 516 len 28672
57572023-12-10T18:52:04.378Z21/30 Write block 275 len 16384 data: 7 7 7 8
57582023-12-10T18:52:04.378Z22/30 Flush
57592023-12-10T18:52:04.385Z23/30 Write block 168 len 12288 data: 7 6 6
57602023-12-10T18:52:04.385Z24/30 Write block 297 len 32768 data: 5 6 6 4 4 5 6 7
57612023-12-10T18:52:04.385Z25/30 Flush
57622023-12-10T18:52:04.399Z26/30 Write block 113 len 8192 data: 4 2
57632023-12-10T18:52:04.399Z27/30 Read block 339 len 16384
57642023-12-10T18:52:04.399Z28/30 Write block 233 len 32768 data: 4 7 6 6 7 8 10 9
57652023-12-10T18:52:04.400Z29/30 Write block 354 len 24576 data: 5 4 5 5 4 4
57662023-12-10T18:52:04.400Z30/30 Read block 384 len 16384
57672023-12-10T18:52:04.400Z----------------------------------------------------------------
57682023-12-10T18:52:04.400Z Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30
57692023-12-10T18:52:04.400ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
57702023-12-10T18:52:04.401Z 1 Acked 1000 Write 7 Done Done Done false
57712023-12-10T18:52:04.401Z 2 Acked 1001 Write 8 Done Done Done false
57722023-12-10T18:52:04.403Z 3 Acked 1002 Write 6 Done Done Done false
57732023-12-10T18:52:04.403Z 4 Acked 1003 Read 4 Sent Done Done false
57742023-12-10T18:52:04.404Z 5 Acked 1004 Write 3 Sent Done Done false
57752023-12-10T18:52:04.404Z 6 Acked 1005 Read 7 Sent Done Done false
57762023-12-10T18:52:04.404Z 7 Acked 1006 Flush 0 Sent Done Done false
57772023-12-10T18:52:04.404Z 8 Acked 1007 Write 8 Sent Done Done false
57782023-12-10T18:52:04.405Z 9 Acked 1008 Read 5 Sent Done Done false
57792023-12-10T18:52:04.405Z 10 Acked 1009 Read 1 Sent Done Done false
57802023-12-10T18:52:04.405Z 11 Acked 1010 Read 1 Sent Done Done false
57812023-12-10T18:52:04.405Z 12 Acked 1011 Read 4 Sent Done Done false
57822023-12-10T18:52:04.405Z 13 Acked 1012 Read 8 Sent Done Done false
57832023-12-10T18:52:04.405Z 14 Acked 1013 Read 6 Sent Done Done false
57842023-12-10T18:52:04.405Z 15 Acked 1014 Write 7 Sent Done Done false
57852023-12-10T18:52:04.405Z 16 Acked 1015 Write 4 Sent Done Done false
57862023-12-10T18:52:04.406Z 17 Acked 1016 Read 1 Sent Done Done false
57872023-12-10T18:52:04.406Z 18 Acked 1017 Read 10 Sent Done Done false
57882023-12-10T18:52:04.406Z 19 Acked 1018 Read 9 Sent Done Done false
57892023-12-10T18:52:04.406Z 20 Acked 1019 Read 7 Sent Done Done false
57902023-12-10T18:52:04.406Z 21 Acked 1020 Write 4 Sent Done Done false
57912023-12-10T18:52:04.407Z 22 Acked 1021 Flush 0 Sent Done Done false
57922023-12-10T18:52:04.408Z 23 Acked 1022 Write 3 Sent Done Done false
57932023-12-10T18:52:04.408Z 24 Acked 1023 Write 8 Sent Done Done false
57942023-12-10T18:52:04.408Z 25 Acked 1024 Flush 0 Sent Done Done false
57952023-12-10T18:52:04.409Z 26 Acked 1025 Write 2 Sent Done Done false
57962023-12-10T18:52:04.409Z 27 Acked 1026 Read 4 Sent Done Done false
57972023-12-10T18:52:04.409Z 28 Acked 1027 Write 8 Sent Done Done false
57982023-12-10T18:52:04.410Z 29 Acked 1028 Write 6 Sent Done Done false
57992023-12-10T18:52:04.410Z 30 Acked 1029 Read 4 Sent Done Sent false
58002023-12-10T18:52:04.410Z STATES DS:0 DS:1 DS:2 TOTAL
58012023-12-10T18:52:04.410Z New 0 0 0 0
58022023-12-10T18:52:04.410Z Sent 27 0 1 28
58032023-12-10T18:52:04.410Z Done 3 30 29 62
58042023-12-10T18:52:04.410Z Skipped 0 0 0 0
58052023-12-10T18:52:04.410Z Error 0 0 0 0
58062023-12-10T18:52:04.411ZLast Flush: 0 1024 1024
58072023-12-10T18:52:04.411ZDownstairs last five completed:
58082023-12-10T18:52:04.411ZUpstairs last five completed: 30 29 28 27 26 25
58092023-12-10T18:52:04.411ZWrote out file "/tmp/test_repair_verify.data"
58102023-12-10T18:52:04.422Z
58112023-12-10T18:52:04.422ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
58122023-12-10T18:52:04.434ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
58132023-12-10T18:52:04.434Z 0 000-019 41 46 46 68 77 77 F F F <---
58142023-12-10T18:52:04.435Z 1 020-039 42 42 42 71 71 71 F F F
58152023-12-10T18:52:04.435Z 2 040-059 44 44 44 74 74 74 F F F
58162023-12-10T18:52:04.435Z 3 060-079 38 38 38 62 62 62 F F F
58172023-12-10T18:52:04.455Z 4 080-099 45 46 46 75 77 77 T F F <---
58182023-12-10T18:52:04.455Z 5 100-119 45 45 45 75 75 75 F T T <---
58192023-12-10T18:52:04.455Z 6 120-139 44 46 46 74 76 76 T F F <---
58202023-12-10T18:52:04.456Z 7 140-159 38 38 38 62 62 62 F F F
58212023-12-10T18:52:04.456Z 8 160-179 43 46 46 73 78 78 F F F <---
58222023-12-10T18:52:04.456Z 9 180-199 43 43 43 73 73 73 F F F
58232023-12-10T18:52:04.456Z 10 200-219 41 46 46 68 76 76 T F F <---
58242023-12-10T18:52:04.457Z 11 220-239 42 46 46 72 76 76 T T T <---
58252023-12-10T18:52:04.457Z 12 240-259 44 44 44 74 74 74 F T T <---
58262023-12-10T18:52:04.457Z 13 260-279 37 46 46 59 77 77 F F F <---
58272023-12-10T18:52:04.457Z 14 280-299 38 46 46 60 78 78 F F F <---
58282023-12-10T18:52:04.457Z 15 300-319 42 46 46 70 78 78 F F F <---
58292023-12-10T18:52:04.457Z 16 320-339 42 42 42 72 72 72 F F F
58302023-12-10T18:52:04.457Z 17 340-359 45 45 45 75 75 75 F T T <---
58312023-12-10T18:52:04.457Z 18 360-379 42 42 42 72 72 72 F F F
58322023-12-10T18:52:04.457Z 19 380-399 41 41 41 68 68 68 F F F
58332023-12-10T18:52:04.457Z 20 400-419 44 44 44 74 74 74 F F F
58342023-12-10T18:52:04.457Z 21 420-439 33 33 33 55 55 55 F F F
58352023-12-10T18:52:04.458Z 22 440-459 44 46 46 74 77 77 F F F <---
58362023-12-10T18:52:04.458Z 23 460-479 44 44 44 74 74 74 F F F
58372023-12-10T18:52:04.458Z 24 480-499 45 45 45 75 75 75 F F F
58382023-12-10T18:52:04.458Z 25 500-519 38 38 38 63 63 63 F F F
58392023-12-10T18:52:04.458Z 26 520-539 44 44 44 74 74 74 F F F
58402023-12-10T18:52:04.458Z 27 540-559 42 42 42 69 69 69 F F F
58412023-12-10T18:52:04.458Z 28 560-579 40 40 40 66 66 66 F F F
58422023-12-10T18:52:04.458Z 29 580-599 38 38 38 61 61 61 F F F
58432023-12-10T18:52:04.458ZMax gen: 46, Max flush: 78
58442023-12-10T18:52:04.458ZError: Difference in extent metadata found!
58452023-12-10T18:52:04.461ZA Difference in extent metadata is expected here
58462023-12-10T18:52:04.462ZOn loop 23
58472023-12-10T18:52:04.462Z
58482023-12-10T18:52:04.462ZVerifying data now
58492023-12-10T18:52:10.716ZNo change after verify
58502023-12-10T18:52:10.716ZLoop: 23 Downstairs dump after verify (and repair):
58512023-12-10T18:52:10.739ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
58522023-12-10T18:52:10.739Z 0 000-019 46 46 46 77 77 77 F F F
58532023-12-10T18:52:10.739Z 1 020-039 42 42 42 71 71 71 F F F
58542023-12-10T18:52:10.739Z 2 040-059 44 44 44 74 74 74 F F F
58552023-12-10T18:52:10.739Z 3 060-079 38 38 38 62 62 62 F F F
58562023-12-10T18:52:10.739Z 4 080-099 46 46 46 77 77 77 F F F
58572023-12-10T18:52:10.739Z 5 100-119 47 47 47 79 79 79 F F F
58582023-12-10T18:52:10.739Z 6 120-139 46 46 46 76 76 76 F F F
58592023-12-10T18:52:10.743Z 7 140-159 38 38 38 62 62 62 F F F
58602023-12-10T18:52:10.743Z 8 160-179 46 46 46 78 78 78 F F F
58612023-12-10T18:52:10.743Z 9 180-199 43 43 43 73 73 73 F F F
58622023-12-10T18:52:10.743Z 10 200-219 46 46 46 76 76 76 F F F
58632023-12-10T18:52:10.744Z 11 220-239 47 47 47 79 79 79 F F F
58642023-12-10T18:52:10.744Z 12 240-259 47 47 47 79 79 79 F F F
58652023-12-10T18:52:10.744Z 13 260-279 46 46 46 77 77 77 F F F
58662023-12-10T18:52:10.744Z 14 280-299 46 46 46 78 78 78 F F F
58672023-12-10T18:52:10.744Z 15 300-319 46 46 46 78 78 78 F F F
58682023-12-10T18:52:10.744Z 16 320-339 42 42 42 72 72 72 F F F
58692023-12-10T18:52:10.744Z 17 340-359 47 47 47 79 79 79 F F F
58702023-12-10T18:52:10.744Z 18 360-379 42 42 42 72 72 72 F F F
58712023-12-10T18:52:10.744Z 19 380-399 41 41 41 68 68 68 F F F
58722023-12-10T18:52:10.744Z 20 400-419 44 44 44 74 74 74 F F F
58732023-12-10T18:52:10.744Z 21 420-439 33 33 33 55 55 55 F F F
58742023-12-10T18:52:10.744Z 22 440-459 46 46 46 77 77 77 F F F
58752023-12-10T18:52:10.744Z 23 460-479 44 44 44 74 74 74 F F F
58762023-12-10T18:52:10.744Z 24 480-499 45 45 45 75 75 75 F F F
58772023-12-10T18:52:10.744Z 25 500-519 38 38 38 63 63 63 F F F
58782023-12-10T18:52:10.744Z 26 520-539 44 44 44 74 74 74 F F F
58792023-12-10T18:52:10.744Z 27 540-559 42 42 42 69 69 69 F F F
58802023-12-10T18:52:10.744Z 28 560-579 40 40 40 66 66 66 F F F
58812023-12-10T18:52:10.745Z 29 580-599 38 38 38 61 61 61 F F F
58822023-12-10T18:52:10.745ZMax gen: 47, Max flush: 79
58832023-12-10T18:52:10.748Z
58842023-12-10T18:52:10.748ZBegin loop 24
58852023-12-10T18:52:10.748ZDownstairs to restart: 1
58862023-12-10T18:52:10.777Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.777745575Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354}
58872023-12-10T18:52:10.777Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.778223541Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354}
58882023-12-10T18:52:10.777Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.778252146Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354}
58892023-12-10T18:52:10.777Z{"msg":"Crucible f7cd171e-c947-41cd-8e68-5bc07247b95b has session id: 13137963-0908-40b8-b278-610781b0c501","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.778273598Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
58902023-12-10T18:52:10.784Z{"msg":"Upstairs opts: Upstairs UUID: f7cd171e-c947-41cd-8e68-5bc07247b95b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.778339332Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
58912023-12-10T18:52:10.784Z{"msg":"Crucible stats registered with UUID: f7cd171e-c947-41cd-8e68-5bc07247b95b","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.778417534Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"Crucible runtime is spawned
58922023-12-10T18:52:10.784Z13137963-0908-40b8-b278-610781b0c501"}
58932023-12-10T18:52:10.784ZThe guest has requested activation with gen:48
58942023-12-10T18:52:10.784Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.77863023Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"looper":"0","session_id":"13137963-0908-40b8-b278-610781b0c501"}
58952023-12-10T18:52:10.784Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.778741326Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"looper":"1","session_id":"13137963-0908-40b8-b278-610781b0c501"}
58962023-12-10T18:52:10.784Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.778787291Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501","task":"up_listen"}
58972023-12-10T18:52:10.784Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.778854702Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
58982023-12-10T18:52:10.784Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.77894834Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
58992023-12-10T18:52:10.785Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.778989825Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"looper":"2","session_id":"13137963-0908-40b8-b278-610781b0c501"}
59002023-12-10T18:52:10.785Z{"msg":"[0] f7cd171e-c947-41cd-8e68-5bc07247b95b looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.779422246Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"looper":"0","session_id":"13137963-0908-40b8-b278-610781b0c501"}
59012023-12-10T18:52:10.785Z{"msg":"f7cd171e-c947-41cd-8e68-5bc07247b95b active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.779495059Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59022023-12-10T18:52:10.785Z{"msg":"[2] f7cd171e-c947-41cd-8e68-5bc07247b95b looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.779522347Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"looper":"2","session_id":"13137963-0908-40b8-b278-610781b0c501"}
59032023-12-10T18:52:10.785Z{"msg":"Set desired generation to :48","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.779547666Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59042023-12-10T18:52:10.785Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.779569013Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59052023-12-10T18:52:10.785Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.779923413Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59062023-12-10T18:52:10.785Z{"msg":"[0] f7cd171e-c947-41cd-8e68-5bc07247b95b (13137963-0908-40b8-b278-610781b0c501) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.780358456Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59072023-12-10T18:52:10.785Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.780463216Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59082023-12-10T18:52:10.785Z{"msg":"[0] client is_active_req TRUE, promote! session 13137963-0908-40b8-b278-610781b0c501","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.780530527Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59092023-12-10T18:52:10.785Z{"msg":"[2] f7cd171e-c947-41cd-8e68-5bc07247b95b (13137963-0908-40b8-b278-610781b0c501) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.780558876Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59102023-12-10T18:52:10.785Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.78057985Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59112023-12-10T18:52:10.785Z{"msg":"[2] client is_active_req TRUE, promote! session 13137963-0908-40b8-b278-610781b0c501","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.780615853Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59122023-12-10T18:52:10.785Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.78120188Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59132023-12-10T18:52:10.785Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.781292895Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59142023-12-10T18:52:10.785Z{"msg":"f7cd171e-c947-41cd-8e68-5bc07247b95b WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.781336258Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59152023-12-10T18:52:10.786Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.781477518Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59162023-12-10T18:52:10.786Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.781604921Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59172023-12-10T18:52:10.786Z{"msg":"f7cd171e-c947-41cd-8e68-5bc07247b95b WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.781667826Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59182023-12-10T18:52:10.786Z{"msg":"[0] f7cd171e-c947-41cd-8e68-5bc07247b95b (13137963-0908-40b8-b278-610781b0c501) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.782131201Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59192023-12-10T18:52:10.786Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.782213041Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59202023-12-10T18:52:10.786Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:10.782257216Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59212023-12-10T18:52:10.786Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.782294917Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59222023-12-10T18:52:10.786Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.782327623Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59232023-12-10T18:52:10.786Z{"msg":"f7cd171e-c947-41cd-8e68-5bc07247b95b WaitQuorum New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.782357592Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59242023-12-10T18:52:10.786Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.782418294Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59252023-12-10T18:52:10.786Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.782459923Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59262023-12-10T18:52:10.786Z{"msg":"[2] f7cd171e-c947-41cd-8e68-5bc07247b95b (13137963-0908-40b8-b278-610781b0c501) WaitQuorum New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.782522676Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59272023-12-10T18:52:10.786Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.782564758Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59282023-12-10T18:52:10.786Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:10.782614756Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59292023-12-10T18:52:10.786Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.782638022Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59302023-12-10T18:52:10.786Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.782659597Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59312023-12-10T18:52:10.786Z{"msg":"f7cd171e-c947-41cd-8e68-5bc07247b95b WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.782701068Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59322023-12-10T18:52:10.787Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.782723645Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59332023-12-10T18:52:10.787Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:10.78274331Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59342023-12-10T18:52:11.780Z{"msg":"[1] f7cd171e-c947-41cd-8e68-5bc07247b95b looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.780822183Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"looper":"1","session_id":"13137963-0908-40b8-b278-610781b0c501"}
59352023-12-10T18:52:11.780Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.780908018Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59362023-12-10T18:52:11.802Z{"msg":"[1] f7cd171e-c947-41cd-8e68-5bc07247b95b (13137963-0908-40b8-b278-610781b0c501) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.781962802Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59372023-12-10T18:52:11.802Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.782019128Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59382023-12-10T18:52:11.802Z{"msg":"[1] client is_active_req TRUE, promote! session 13137963-0908-40b8-b278-610781b0c501","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.782044718Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59392023-12-10T18:52:11.802Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.782666244Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59402023-12-10T18:52:11.802Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.782725732Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59412023-12-10T18:52:11.802Z{"msg":"f7cd171e-c947-41cd-8e68-5bc07247b95b WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.782752028Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59422023-12-10T18:52:11.802Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.783365653Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59432023-12-10T18:52:11.803Z{"msg":"[1] f7cd171e-c947-41cd-8e68-5bc07247b95b (13137963-0908-40b8-b278-610781b0c501) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.783503993Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59442023-12-10T18:52:11.803Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.783572721Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59452023-12-10T18:52:11.803Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:11.783596748Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59462023-12-10T18:52:11.803Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.783616967Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59472023-12-10T18:52:11.803Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.783677019Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59482023-12-10T18:52:11.803Z{"msg":"f7cd171e-c947-41cd-8e68-5bc07247b95b WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.783704866Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59492023-12-10T18:52:11.803Z{"msg":"[0]R flush_numbers[0..12]: [77, 71, 74, 62, 77, 79, 76, 62, 78, 73, 76, 79]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.783763977Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59502023-12-10T18:52:11.803Z{"msg":"[0]R generation[0..12]: [46, 42, 44, 38, 46, 47, 46, 38, 46, 43, 46, 47]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.78379532Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59512023-12-10T18:52:11.803Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.783891948Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59522023-12-10T18:52:11.803Z{"msg":"[1]R flush_numbers[0..12]: [77, 71, 74, 62, 77, 79, 76, 62, 78, 73, 76, 79]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.783918856Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","The guest has finished waiting for activation with:48
59532023-12-10T18:52:11.803Zpid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59542023-12-10T18:52:11.803ZWait for a query_work_queue command to finish before sending IO
59552023-12-10T18:52:11.803Z{"msg":"[1]R generation[0..12]: [46, 42, 44, 38, 46, 47, 46, 38, 46, 43, 46, 47]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.783959699Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59562023-12-10T18:52:11.803Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.78401745Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59572023-12-10T18:52:11.803Z{"msg":"[2]R flush_numbers[0..12]: [77, 71, 74, 62, 77, 79, 76, 62, 78, 73, 76, 79]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.784043707Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59582023-12-10T18:52:11.803Z{"msg":"[2]R generation[0..12]: [46, 42, 44, 38, 46, 47, 46, 38, 46, 43, 46, 47]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.784126954Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59592023-12-10T18:52:11.803Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.784160933Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59602023-12-10T18:52:11.806Z{"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.784182585Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59612023-12-10T18:52:11.806Z{"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.784202946Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59622023-12-10T18:52:11.806Z{"msg":"Next flush: 80","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.784236069Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59632023-12-10T18:52:11.806Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.784297668Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59642023-12-10T18:52:11.806Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.784323011Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59652023-12-10T18:52:11.806Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.78434631Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59662023-12-10T18:52:11.806Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.784382782Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59672023-12-10T18:52:11.806Z{"msg":"f7cd171e-c947-41cd-8e68-5bc07247b95b is now active with session: 13137963-0908-40b8-b278-610781b0c501","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.784408551Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59682023-12-10T18:52:11.806Z{"msg":"f7cd171e-c947-41cd-8e68-5bc07247b95b Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.784431897Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59692023-12-10T18:52:11.806Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.784492817Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59702023-12-10T18:52:11.806Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.784521169Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59712023-12-10T18:52:11.807Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.784577851Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59722023-12-10T18:52:11.807Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.784602956Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59732023-12-10T18:52:11.808Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.784665209Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59742023-12-10T18:52:11.808Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.784690218Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59752023-12-10T18:52:11.808Z{"msg":"[0] All repairs completed, exit"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
59762023-12-10T18:52:11.818Z,"v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.784836875Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59772023-12-10T18:52:11.818Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.784892725Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59782023-12-10T18:52:11.818Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.784916566Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59792023-12-10T18:52:11.820Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.784974861Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59802023-12-10T18:52:11.820Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:11.785003295Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1354,"session_id":"13137963-0908-40b8-b278-610781b0c501"}
59812023-12-10T18:52:11.820ZLoading write count information from file "/tmp/test_repair_verify.data"
59822023-12-10T18:52:11.820ZRun Repair workload
59832023-12-10T18:52:11.820Z01/30 Write block 248 len 36864 data: 5 7 6 6 5 6 3 3 3
59842023-12-10T18:52:11.820Z02/30 Flush
59852023-12-10T18:52:11.820Z03/30 Write block 183 len 4096 data: 5
59862023-12-10T18:52:11.820Z04/30 Read block 356 len 8192
59872023-12-10T18:52:11.820Z05/30 Read block 432 len 12288
59882023-12-10T18:52:11.820Z06/30 Read block 500 len 28672
59892023-12-10T18:52:11.820Z07/30 Write block 156 len 4096 data: 3
59902023-12-10T18:52:11.820Z08/30 Read block 490 len 36864
59912023-12-10T18:52:11.820Z09/30 Write block 8 len 8192 data: 3 3
59922023-12-10T18:52:11.820Z10/30 Read block 309 len 12288
59932023-12-10T18:52:11.821Z11/30 Read block 189 len 32768
59942023-12-10T18:52:11.821Z12/30 Read block 564 len 40960
59952023-12-10T18:52:11.821Z13/30 Write block 471 len 24576 data: 9 8 7 5 4 4
59962023-12-10T18:52:11.821Z14/30 Write block 398 len 24576 data: 4 5 5 5 5 5
59972023-12-10T18:52:11.821Z15/30 Flush
59982023-12-10T18:52:11.825Z16/30 Write block 15 len 20480 data: 4 4 6 4 4
59992023-12-10T18:52:11.825Z17/30 Write block 403 len 12288 data: 6 5 5
60002023-12-10T18:52:11.825Z18/30 Write block 474 len 16384 data: 6 5 5 3
60012023-12-10T18:52:11.825Z19/30 Read block 173 len 28672
60022023-12-10T18:52:11.825Z20/30 Read block 426 len 40960
60032023-12-10T18:52:11.826Z21/30 Read block 35 len 24576
60042023-12-10T18:52:11.829Z22/30 Write block 371 len 40960 data: 3 4 2 3 5 5 5 7 7 7
60052023-12-10T18:52:11.829Z23/30 Write block 111 len 32768 data: 4 4 5 3 4 5 6 7
60062023-12-10T18:52:11.829Z24/30 Write block 56 len 36864 data: 4 3 3 3 2 2 2 2 2
60072023-12-10T18:52:11.829Z25/30 Read block 501 len 24576
60082023-12-10T18:52:11.834Z26/30 Read block 59 len 4096
60092023-12-10T18:52:11.849Z27/30 Write block 249 len 40960 data: 8 7 7 6 7 4 4 4 2 2
60102023-12-10T18:52:11.849Z28/30 Read block 323 len 24576
60112023-12-10T18:52:11.849Z29/30 Write block 421 len 20480 data: 5 5 5 6 4
60122023-12-10T18:52:11.849Z30/30 Read block 67 len 4096
60132023-12-10T18:52:11.884Z----------------------------------------------------------------
60142023-12-10T18:52:11.885Z Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30
60152023-12-10T18:52:11.885ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
60162023-12-10T18:52:11.885Z 1 Acked 1000 Write 9 Done Sent Done false
60172023-12-10T18:52:11.885Z 2 Acked 1001 Flush 0 Done Sent Done false
60182023-12-10T18:52:11.885Z 3 Acked 1002 Write 1 Done Sent Done false
60192023-12-10T18:52:11.885Z 4 Acked 1003 Read 2 Done Sent Done false
60202023-12-10T18:52:11.885Z 5 Acked 1004 Read 3 Done Sent Done false
60212023-12-10T18:52:11.885Z 6 Acked 1005 Read 7 Done Sent Done false
60222023-12-10T18:52:11.885Z 7 Acked 1006 Write 1 Done Sent Done false
60232023-12-10T18:52:11.885Z 8 Acked 1007 Read 9 Done Sent Done false
60242023-12-10T18:52:11.885Z 9 Acked 1008 Write 2 Done Sent Done false
60252023-12-10T18:52:11.885Z 10 Acked 1009 Read 3 Done Sent Done false
60262023-12-10T18:52:11.885Z 11 Acked 1010 Read 8 Done Sent Done false
60272023-12-10T18:52:11.885Z 12 Acked 1011 Read 10 Done Sent Done false
60282023-12-10T18:52:11.886Z 13 Acked 1012 Write 6 Done Sent Done false
60292023-12-10T18:52:11.886Z 14 Acked 1013 Write 6 Done Sent Done false
60302023-12-10T18:52:11.886Z 15 Acked 1014 Flush 0 Done Sent Done false
60312023-12-10T18:52:11.886Z 16 Acked 1015 Write 5 Done Sent Done false
60322023-12-10T18:52:11.886Z 17 Acked 1016 Write 3 Done Sent Done false
60332023-12-10T18:52:11.886Z 18 Acked 1017 Write 4 Done Sent Done false
60342023-12-10T18:52:11.886Z 19 Acked 1018 Read 7 Done Sent Done false
60352023-12-10T18:52:11.886Z 20 Acked 1019 Read 10 Done Sent Done false
60362023-12-10T18:52:11.886Z 21 Acked 1020 Read 6 Done Sent Done false
60372023-12-10T18:52:11.886Z 22 Acked 1021 Write 10 Done Sent Done false
60382023-12-10T18:52:11.886Z 23 Acked 1022 Write 8 Done Sent Done false
60392023-12-10T18:52:11.886Z 24 Acked 1023 Write 9 Done Sent Done false
60402023-12-10T18:52:11.886Z 25 Acked 1024 Read 6 Done Sent Done false
60412023-12-10T18:52:11.886Z 26 Acked 1025 Read 1 Done Sent Done false
60422023-12-10T18:52:11.886Z 27 Acked 1026 Write 10 Done Sent Done false
60432023-12-10T18:52:11.886Z 28 Acked 1027 Read 6 Done Sent Done false
60442023-12-10T18:52:11.886Z 29 Acked 1028 Write 5 Done Sent Done false
60452023-12-10T18:52:11.886Z 30 Acked 1029 Read 1 Done Sent Done false
60462023-12-10T18:52:11.887Z STATES DS:0 DS:1 DS:2 TOTAL
60472023-12-10T18:52:11.887Z New 0 0 0 0
60482023-12-10T18:52:11.887Z Sent 0 30 0 30
60492023-12-10T18:52:11.887Z Done 30 0 30 60
60502023-12-10T18:52:11.887Z Skipped 0 0 0 0
60512023-12-10T18:52:11.887Z Error 0 0 0 0
60522023-12-10T18:52:11.887ZLast Flush: 1014 0 1014
60532023-12-10T18:52:11.887ZDownstairs last five completed:
60542023-12-10T18:52:11.887ZUpstairs last five completed: 30 29 28 27 26 25
60552023-12-10T18:52:11.887ZWrote out file "/tmp/test_repair_verify.data"
60562023-12-10T18:52:11.887Z
60572023-12-10T18:52:11.896ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
60582023-12-10T18:52:11.896ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
60592023-12-10T18:52:11.896Z 0 000-019 48 46 48 81 77 81 T F T <---
60602023-12-10T18:52:11.896Z 1 020-039 42 42 42 71 71 71 F F F
60612023-12-10T18:52:11.896Z 2 040-059 44 44 44 74 74 74 T F T <---
60622023-12-10T18:52:11.896Z 3 060-079 38 38 38 62 62 62 T F T <---
60632023-12-10T18:52:11.896Z 4 080-099 46 46 46 77 77 77 F F F
60642023-12-10T18:52:11.896Z 5 100-119 47 47 47 79 79 79 T F T <---
60652023-12-10T18:52:11.896Z 6 120-139 46 46 46 76 76 76 F F F
60662023-12-10T18:52:11.896Z 7 140-159 48 38 48 81 62 81 F F F <---
60672023-12-10T18:52:11.897Z 8 160-179 46 46 46 78 78 78 F F F
60682023-12-10T18:52:11.897Z 9 180-199 48 43 48 81 73 81 F F F <---
60692023-12-10T18:52:11.897Z 10 200-219 46 46 46 76 76 76 F F F
60702023-12-10T18:52:11.897Z 11 220-239 47 47 47 79 79 79 F F F
60712023-12-10T18:52:11.897Z 12 240-259 48 47 48 80 79 80 T F T <---
60722023-12-10T18:52:11.897Z 13 260-279 46 46 46 77 77 77 F F F
60732023-12-10T18:52:11.897Z 14 280-299 46 46 46 78 78 78 F F F
60742023-12-10T18:52:11.897Z 15 300-319 46 46 46 78 78 78 F F F
60752023-12-10T18:52:11.897Z 16 320-339 42 42 42 72 72 72 F F F
60762023-12-10T18:52:11.897Z 17 340-359 47 47 47 79 79 79 F F F
60772023-12-10T18:52:11.897Z 18 360-379 42 42 42 72 72 72 T F T <---
60782023-12-10T18:52:11.898Z 19 380-399 48 41 48 81 68 81 T F T <---
60792023-12-10T18:52:11.898Z 20 400-419 48 44 48 81 74 81 T F T <---
60802023-12-10T18:52:11.898Z 21 420-439 33 33 33 55 55 55 T F T <---
60812023-12-10T18:52:11.898Z 22 440-459 46 46 46 77 77 77 F F F
60822023-12-10T18:52:11.898Z 23 460-479 48 44 48 81 74 81 T F T <---
60832023-12-10T18:52:11.898Z 24 480-499 45 45 45 75 75 75 F F F
60842023-12-10T18:52:11.898Z 25 500-519 38 38 38 63 63 63 F F F
60852023-12-10T18:52:11.899Z 26 520-539 44 44 44 74 74 74 F F F
60862023-12-10T18:52:11.899Z 27 540-559 42 42 42 69 69 69 F F F
60872023-12-10T18:52:11.899Z 28 560-579 40 40 40 66 66 66 F F F
60882023-12-10T18:52:11.900Z 29 580-599 38 38 38 61 61 61 F F F
60892023-12-10T18:52:11.900ZMax gen: 48, Max flush: 81
60902023-12-10T18:52:11.900ZError: Difference in extent metadata found!
60912023-12-10T18:52:11.900ZA Difference in extent metadata is expected here
60922023-12-10T18:52:11.900ZOn loop 24
60932023-12-10T18:52:11.900Z
60942023-12-10T18:52:11.907ZVerifying data now
60952023-12-10T18:52:17.131ZNo change after verify
60962023-12-10T18:52:17.131ZLoop: 24 Downstairs dump after verify (and repair):
60972023-12-10T18:52:17.151ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
60982023-12-10T18:52:17.151Z 0 000-019 49 49 49 82 82 82 F F F
60992023-12-10T18:52:17.178Z 1 020-039 42 42 42 71 71 71 F F F
61002023-12-10T18:52:17.178Z 2 040-059 49 49 49 82 82 82 F F F
61012023-12-10T18:52:17.178Z 3 060-079 49 49 49 82 82 82 F F F
61022023-12-10T18:52:17.179Z 4 080-099 46 46 46 77 77 77 F F F
61032023-12-10T18:52:17.179Z 5 100-119 49 49 49 82 82 82 F F F
61042023-12-10T18:52:17.179Z 6 120-139 46 46 46 76 76 76 F F F
61052023-12-10T18:52:17.179Z 7 140-159 48 48 48 81 81 81 F F F
61062023-12-10T18:52:17.179Z 8 160-179 46 46 46 78 78 78 F F F
61072023-12-10T18:52:17.179Z 9 180-199 48 48 48 81 81 81 F F F
61082023-12-10T18:52:17.179Z 10 200-219 46 46 46 76 76 76 F F F
61092023-12-10T18:52:17.179Z 11 220-239 47 47 47 79 79 79 F F F
61102023-12-10T18:52:17.180Z 12 240-259 49 49 49 82 82 82 F F F
61112023-12-10T18:52:17.180Z 13 260-279 46 46 46 77 77 77 F F F
61122023-12-10T18:52:17.180Z 14 280-299 46 46 46 78 78 78 F F F
61132023-12-10T18:52:17.180Z 15 300-319 46 46 46 78 78 78 F F F
61142023-12-10T18:52:17.180Z 16 320-339 42 42 42 72 72 72 F F F
61152023-12-10T18:52:17.180Z 17 340-359 47 47 47 79 79 79 F F F
61162023-12-10T18:52:17.180Z 18 360-379 49 49 49 82 82 82 F F F
61172023-12-10T18:52:17.180Z 19 380-399 49 49 49 82 82 82 F F F
61182023-12-10T18:52:17.181Z 20 400-419 49 49 49 82 82 82 F F F
61192023-12-10T18:52:17.182Z 21 420-439 49 49 49 82 82 82 F F F
61202023-12-10T18:52:17.182Z 22 440-459 46 46 46 77 77 77 F F F
61212023-12-10T18:52:17.182Z 23 460-479 49 49 49 82 82 82 F F F
61222023-12-10T18:52:17.182Z 24 480-499 45 45 45 75 75 75 F F F
61232023-12-10T18:52:17.182Z 25 500-519 38 38 38 63 63 63 F F F
61242023-12-10T18:52:17.182Z 26 520-539 44 44 44 74 74 74 F F F
61252023-12-10T18:52:17.182Z 27 540-559 42 42 42 69 69 69 F F F
61262023-12-10T18:52:17.183Z 28 560-579 40 40 40 66 66 66 F F F
61272023-12-10T18:52:17.183Z 29 580-599 38 38 38 61 61 61 F F F
61282023-12-10T18:52:17.183ZMax gen: 49, Max flush: 82
61292023-12-10T18:52:17.183Z
61302023-12-10T18:52:17.183ZBegin loop 25
61312023-12-10T18:52:17.183ZDownstairs to restart: 2
61322023-12-10T18:52:17.205Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.178886567Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362}
61332023-12-10T18:52:17.206Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.179303939Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362}
61342023-12-10T18:52:17.206Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.179332808Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362}
61352023-12-10T18:52:17.206Z{"msg":"Crucible 620c93ba-f7d2-4ec1-a07e-8f1a0851b2db has session id: 9f57fbb5-65b8-4d7b-b34a-8f158590cc22","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.179395912Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61362023-12-10T18:52:17.206Z{"msg":"Upstairs opts: Upstairs UUID: 620c93ba-f7d2-4ec1-a07e-8f1a0851b2db, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.17950303Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61372023-12-10T18:52:17.206Z{"msg":"Crucible stats registered with UUID: 620c93ba-f7d2-4ec1-a07e-8f1a0851b2db","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.179583091Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61382023-12-10T18:52:17.206ZCrucible runtime is spawned
61392023-12-10T18:52:17.206ZThe guest has requested activation with gen:50
61402023-12-10T18:52:17.206Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.179868388Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22","task":"up_listen"}
61412023-12-10T18:52:17.206Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.179931687Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"looper":"2","session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61422023-12-10T18:52:17.206Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.179965094Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61432023-12-10T18:52:17.206Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.180010045Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61442023-12-10T18:52:17.206Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.180070767Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"looper":"0","session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61452023-12-10T18:52:17.206Z{"msg":"[2] 620c93ba-f7d2-4ec1-a07e-8f1a0851b2db looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.180175844Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"looper":"2","session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61462023-12-10T18:52:17.206Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.180251585Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"looper":"1","session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61472023-12-10T18:52:17.206Z{"msg":"[0] 620c93ba-f7d2-4ec1-a07e-8f1a0851b2db looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.18029497Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"looper":"0","session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61482023-12-10T18:52:17.207Z{"msg":"[1] 620c93ba-f7d2-4ec1-a07e-8f1a0851b2db looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.180452347Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"looper":"1","session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61492023-12-10T18:52:17.207Z{"msg":"620c93ba-f7d2-4ec1-a07e-8f1a0851b2db active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.180876176Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61502023-12-10T18:52:17.207Z{"msg":"Set desired generation to :50","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.180937878Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61512023-12-10T18:52:17.207Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.181048395Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61522023-12-10T18:52:17.207Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.18133912Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61532023-12-10T18:52:17.208Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.18139108Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61542023-12-10T18:52:17.208Z{"msg":"[0] 620c93ba-f7d2-4ec1-a07e-8f1a0851b2db (9f57fbb5-65b8-4d7b-b34a-8f158590cc22) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.181836294Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61552023-12-10T18:52:17.208Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.181997579Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61562023-12-10T18:52:17.208Z{"msg":"[0] client is_active_req TRUE, promote! session 9f57fbb5-65b8-4d7b-b34a-8f158590cc22","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.182136991Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61572023-12-10T18:52:17.208Z{"msg":"[1] 620c93ba-f7d2-4ec1-a07e-8f1a0851b2db (9f57fbb5-65b8-4d7b-b34a-8f158590cc22) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.182166062Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61582023-12-10T18:52:17.208Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.182200101Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61592023-12-10T18:52:17.208Z{"msg":"[1] client is_active_req TRUE, promote! session 9f57fbb5-65b8-4d7b-b34a-8f158590cc22","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.182240199Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61602023-12-10T18:52:17.208Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.182632372Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61612023-12-10T18:52:17.208Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.182740361Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61622023-12-10T18:52:17.208Z{"msg":"620c93ba-f7d2-4ec1-a07e-8f1a0851b2db WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.18280204Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61632023-12-10T18:52:17.208Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.182847305Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61642023-12-10T18:52:17.208Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.1829087Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61652023-12-10T18:52:17.208Z{"msg":"620c93ba-f7d2-4ec1-a07e-8f1a0851b2db WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.183009043Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61662023-12-10T18:52:17.208Z{"msg":"[2] 620c93ba-f7d2-4ec1-a07e-8f1a0851b2db (9f57fbb5-65b8-4d7b-b34a-8f158590cc22) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.183295279Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61672023-12-10T18:52:17.208Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.183371383Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61682023-12-10T18:52:17.208Z{"msg":"[2] client is_active_req TRUE, promote! session 9f57fbb5-65b8-4d7b-b34a-8f158590cc22","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.183415667Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61692023-12-10T18:52:17.208Z{"msg":"[0] 620c93ba-f7d2-4ec1-a07e-8f1a0851b2db (9f57fbb5-65b8-4d7b-b34a-8f158590cc22) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.183463121Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61702023-12-10T18:52:17.209Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.183612967Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61712023-12-10T18:52:17.209Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:17.183682755Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61722023-12-10T18:52:17.209Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.183709668Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61732023-12-10T18:52:17.209Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.183767722Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61742023-12-10T18:52:17.209Z{"msg":"620c93ba-f7d2-4ec1-a07e-8f1a0851b2db WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.183799135Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61752023-12-10T18:52:17.209Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.183863412Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61762023-12-10T18:52:17.209Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.183887317Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61772023-12-10T18:52:17.210Z{"msg":"[1] 620c93ba-f7d2-4ec1-a07e-8f1a0851b2db (9f57fbb5-65b8-4d7b-b34a-8f158590cc22) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.183919172Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61782023-12-10T18:52:17.210Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.183969567Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61792023-12-10T18:52:17.210Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:17.183991052Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61802023-12-10T18:52:17.210Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.184052585Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61812023-12-10T18:52:17.210Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.184080743Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61822023-12-10T18:52:17.211Z{"msg":"620c93ba-f7d2-4ec1-a07e-8f1a0851b2db WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.184136106Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61832023-12-10T18:52:17.211Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.184161375Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61842023-12-10T18:52:17.211Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.184181217Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61852023-12-10T18:52:17.211Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.184238621Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61862023-12-10T18:52:17.211Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.184275381Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61872023-12-10T18:52:17.211Z{"msg":"620c93ba-f7d2-4ec1-a07e-8f1a0851b2db WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.184330386Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61882023-12-10T18:52:17.211Z{"msg":"[2] 620c93ba-f7d2-4ec1-a07e-8f1a0851b2db (9f57fbb5-65b8-4d7b-b34a-8f158590cc22) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.184733621Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61892023-12-10T18:52:17.211Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.184822325Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61902023-12-10T18:52:17.211Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:17.18486238Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61912023-12-10T18:52:17.211Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.184918735Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61922023-12-10T18:52:17.211Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.184958566Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61932023-12-10T18:52:17.211Z{"msg":"620c93ba-f7d2-4ec1-a07e-8f1a0851b2db WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.185018305Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61942023-12-10T18:52:17.211Z{"msg":"[0]R flush_numbers[0..12]: [82, 71, 82, 82, 77, 82, 76, 81, 78, 81, 76, 79]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.185059102Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61952023-12-10T18:52:17.211Z{"msg":"[0]R generation[0..12]: [49, 42, 49, 49, 46, 49, 46, 48, 46, 48, 46, 47]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.185178068Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61962023-12-10T18:52:17.211Z{"msg":"The guest has finished waiting for activation with:50
61972023-12-10T18:52:17.211Z[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO
61982023-12-10T18:52:17.211Z30,"time":"2023-12-10T18:52:17.185274808Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
61992023-12-10T18:52:17.212Z{"msg":"[1]R flush_numbers[0..12]: [82, 71, 82, 82, 77, 82, 76, 81, 78, 81, 76, 79]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.185310168Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
62002023-12-10T18:52:17.212Z{"msg":"[1]R generation[0..12]: [49, 42, 49, 49, 46, 49, 46, 48, 46, 48, 46, 47]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.185342213Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
62012023-12-10T18:52:17.212Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.185430592Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
62022023-12-10T18:52:17.212Z{"msg":"[2]R flush_numbers[0..12]: [82, 71, 82, 82, 77, 82, 76, 81, 78, 81, 76, 79]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.185490774Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
62032023-12-10T18:52:17.212Z{"msg":"[2]R generation[0..12]: [49, 42, 49, 49, 46, 49, 46, 48, 46, 48, 46, 47]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.185516824Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
62042023-12-10T18:52:17.212Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.185544951Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
62052023-12-10T18:52:17.212Z{"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.185603181Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
62062023-12-10T18:52:17.212Z{"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.185628262Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
62072023-12-10T18:52:17.212Z{"msg":"Next flush: 83","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.185655332Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
62082023-12-10T18:52:17.212Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.185692733Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
62092023-12-10T18:52:17.212Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.185729624Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
62102023-12-10T18:52:17.212Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.185761355Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
62112023-12-10T18:52:17.212Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.185794556Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
62122023-12-10T18:52:17.212Z{"msg":"620c93ba-f7d2-4ec1-a07e-8f1a0851b2db is now active with session: 9f57fbb5-65b8-4d7b-b34a-8f158590cc22","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.185851015Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
62132023-12-10T18:52:17.212Z{"msg":"620c93ba-f7d2-4ec1-a07e-8f1a0851b2db Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.185877452Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
62142023-12-10T18:52:17.212Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.185898378Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
62152023-12-10T18:52:17.212Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.185945882Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
62162023-12-10T18:52:17.212Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.18596821Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
62172023-12-10T18:52:17.213Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.185988232Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
62182023-12-10T18:52:17.213Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.186028246Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
62192023-12-10T18:52:17.213Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.1860501Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
62202023-12-10T18:52:17.213Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.186069724Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
62212023-12-10T18:52:17.213Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.186116046Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
62222023-12-10T18:52:17.213ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
62232023-12-10T18:52:17.213Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.186164147Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
62242023-12-10T18:52:17.213Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.186210302Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
62252023-12-10T18:52:17.213Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:17.186233018Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1362,"session_id":"9f57fbb5-65b8-4d7b-b34a-8f158590cc22"}
62262023-12-10T18:52:17.213ZLoading write count information from file "/tmp/test_repair_verify.data"
62272023-12-10T18:52:17.213ZRun Repair workload
62282023-12-10T18:52:17.213Z01/30 Write block 578 len 28672 data: 5 5 5 5 4 4 3
62292023-12-10T18:52:17.213Z02/30 Write block 240 len 20480 data: 10 9 9 9 9
62302023-12-10T18:52:17.213Z03/30 Write block 379 len 28672 data: 8 8 5 6 6 7 7
62312023-12-10T18:52:17.213Z04/30 Read block 215 len 4096
62322023-12-10T18:52:17.224Z05/30 Read block 418 len 40960
62332023-12-10T18:52:17.224Z06/30 Write block 217 len 24576 data: 9 7 6 6 8 8
62342023-12-10T18:52:17.224Z07/30 Read block 473 len 16384
62352023-12-10T18:52:17.224Z08/30 Write block 133 len 28672 data: 2 2 2 2 2 2 2
62362023-12-10T18:52:17.224Z09/30 Read block 162 len 8192
62372023-12-10T18:52:17.224Z10/30 Read block 275 len 20480
62382023-12-10T18:52:17.224Z11/30 Read block 280 len 36864
62392023-12-10T18:52:17.224Z12/30 Read block 327 len 16384
62402023-12-10T18:52:17.224Z13/30 Write block 154 len 4096 data: 3
62412023-12-10T18:52:17.224Z14/30 Read block 594 len 16384
62422023-12-10T18:52:17.224Z15/30 Read block 440 len 20480
62432023-12-10T18:52:17.224Z16/30 Write block 239 len 20480 data: 11 11 10 10 10
62442023-12-10T18:52:17.224Z17/30 Read block 332 len 4096
62452023-12-10T18:52:17.224Z18/30 Read block 447 len 40960
62462023-12-10T18:52:17.225Z19/30 Read block 403 len 40960
62472023-12-10T18:52:17.233Z20/30 Write block 563 len 32768 data: 4 4 4 4 3 3 3 4
62482023-12-10T18:52:17.233Z21/30 Read block 103 len 28672
62492023-12-10T18:52:17.234Z22/30 Write block 246 len 16384 data: 8 7 6 9
62502023-12-10T18:52:17.234Z23/30 Flush
62512023-12-10T18:52:17.238Z24/30 Write block 234 len 36864 data: 8 7 7 8 9 12 12 11 11
62522023-12-10T18:52:17.238Z25/30 Write block 64 len 12288 data: 3 4 5
62532023-12-10T18:52:17.238Z26/30 Read block 176 len 32768
62542023-12-10T18:52:17.238Z27/30 Write block 38 len 8192 data: 4 5
62552023-12-10T18:52:17.239Z28/30 Write block 326 len 12288 data: 4 4 4
62562023-12-10T18:52:17.253Z29/30 Read block 248 len 32768
62572023-12-10T18:52:17.253Z30/30 Write block 71 len 40960 data: 5 4 4 6 6 6 6 6 7 8
62582023-12-10T18:52:17.253Z----------------------------------------------------------------
62592023-12-10T18:52:17.253Z Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:30
62602023-12-10T18:52:17.254ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
62612023-12-10T18:52:17.254Z 1 Acked 1000 Write 7 Done Done Done false
62622023-12-10T18:52:17.254Z 2 Acked 1001 Write 5 Done Done Done false
62632023-12-10T18:52:17.254Z 3 Acked 1002 Write 7 Done Done Done false
62642023-12-10T18:52:17.254Z 4 Acked 1003 Read 1 Done Done Done false
62652023-12-10T18:52:17.254Z 5 Acked 1004 Read 10 Done Done Done false
62662023-12-10T18:52:17.254Z 6 Acked 1005 Write 6 Done Done Done false
62672023-12-10T18:52:17.254Z 7 Acked 1006 Read 4 Done Done Done false
62682023-12-10T18:52:17.254Z 8 Acked 1007 Write 7 Done Done Done false
62692023-12-10T18:52:17.254Z 9 Acked 1008 Read 2 Done Done Done false
62702023-12-10T18:52:17.254Z 10 Acked 1009 Read 5 Done Done Done false
62712023-12-10T18:52:17.254Z 11 Acked 1010 Read 9 Done Done Done false
62722023-12-10T18:52:17.254Z 12 Acked 1011 Read 4 Done Done Sent false
62732023-12-10T18:52:17.254Z 13 Acked 1012 Write 1 Done Done Sent false
62742023-12-10T18:52:17.254Z 14 Acked 1013 Read 4 Done Done Sent false
62752023-12-10T18:52:17.254Z 15 Acked 1014 Read 5 Done Done Sent false
62762023-12-10T18:52:17.255Z 16 Acked 1015 Write 5 Done Done Sent false
62772023-12-10T18:52:17.255Z 17 Acked 1016 Read 1 Done Done Sent false
62782023-12-10T18:52:17.255Z 18 Acked 1017 Read 10 Done Done Sent false
62792023-12-10T18:52:17.255Z 19 Acked 1018 Read 10 Done Done Sent false
62802023-12-10T18:52:17.255Z 20 Acked 1019 Write 8 Done Done Sent false
62812023-12-10T18:52:17.255Z 21 Acked 1020 Read 7 Done Done Sent false
62822023-12-10T18:52:17.255Z 22 Acked 1021 Write 4 Done Done Sent false
62832023-12-10T18:52:17.255Z 23 Acked 1022 Flush 0 Done Done Sent false
62842023-12-10T18:52:17.255Z 24 Acked 1023 Write 9 Done Done Sent false
62852023-12-10T18:52:17.255Z 25 Acked 1024 Write 3 Done Done Sent false
62862023-12-10T18:52:17.256Z 26 Acked 1025 Read 8 Done Done Sent false
62872023-12-10T18:52:17.257Z 27 Acked 1026 Write 2 Done Done Sent false
62882023-12-10T18:52:17.257Z 28 Acked 1027 Write 3 Done Done Sent false
62892023-12-10T18:52:17.257Z 29 Acked 1028 Read 8 Done Done Sent false
62902023-12-10T18:52:17.257Z 30 Acked 1029 Write 10 Sent Sent Sent false
62912023-12-10T18:52:17.257Z STATES DS:0 DS:1 DS:2 TOTAL
62922023-12-10T18:52:17.257Z New 0 0 0 0
62932023-12-10T18:52:17.257Z Sent 1 1 19 21
62942023-12-10T18:52:17.257Z Done 29 29 11 69
62952023-12-10T18:52:17.257Z Skipped 0 0 0 0
62962023-12-10T18:52:17.257Z Error 0 0 0 0
62972023-12-10T18:52:17.257ZLast Flush: 1022 1022 0
62982023-12-10T18:52:17.257ZDownstairs last five completed:
62992023-12-10T18:52:17.257ZUpstairs last five completed: 30 29 28 27 26 25
63002023-12-10T18:52:17.257ZWrote out file "/tmp/test_repair_verify.data"
63012023-12-10T18:52:17.257Z
63022023-12-10T18:52:17.261ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
63032023-12-10T18:52:17.287ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
63042023-12-10T18:52:17.288Z 0 000-019 49 49 49 82 82 82 F F F
63052023-12-10T18:52:17.291Z 1 020-039 42 42 42 71 71 71 T T F <---
63062023-12-10T18:52:17.291Z 2 040-059 49 49 49 82 82 82 F F F
63072023-12-10T18:52:17.291Z 3 060-079 49 49 49 82 82 82 T T F <---
63082023-12-10T18:52:17.291Z 4 080-099 46 46 46 77 77 77 T T F <---
63092023-12-10T18:52:17.291Z 5 100-119 49 49 49 82 82 82 F F F
63102023-12-10T18:52:17.291Z 6 120-139 50 50 46 83 83 76 F F T <---
63112023-12-10T18:52:17.291Z 7 140-159 50 50 48 83 83 81 F F F <---
63122023-12-10T18:52:17.291Z 8 160-179 46 46 46 78 78 78 F F F
63132023-12-10T18:52:17.291Z 9 180-199 48 48 48 81 81 81 F F F
63142023-12-10T18:52:17.292Z 10 200-219 50 50 46 83 83 76 F F T <---
63152023-12-10T18:52:17.292Z 11 220-239 50 50 47 83 83 79 T T T <---
63162023-12-10T18:52:17.292Z 12 240-259 50 50 49 83 83 82 T T T <---
63172023-12-10T18:52:17.292Z 13 260-279 46 46 46 77 77 77 F F F
63182023-12-10T18:52:17.292Z 14 280-299 46 46 46 78 78 78 F F F
63192023-12-10T18:52:17.292Z 15 300-319 46 46 46 78 78 78 F F F
63202023-12-10T18:52:17.292Z 16 320-339 42 42 42 72 72 72 T T F <---
63212023-12-10T18:52:17.292Z 17 340-359 47 47 47 79 79 79 F F F
63222023-12-10T18:52:17.292Z 18 360-379 50 50 49 83 83 82 F F T <---
63232023-12-10T18:52:17.292Z 19 380-399 50 50 49 83 83 82 F F T <---
63242023-12-10T18:52:17.292Z 20 400-419 49 49 49 82 82 82 F F F
63252023-12-10T18:52:17.292Z 21 420-439 49 49 49 82 82 82 F F F
63262023-12-10T18:52:17.292Z 22 440-459 46 46 46 77 77 77 F F F
63272023-12-10T18:52:17.294Z 23 460-479 49 49 49 82 82 82 F F F
63282023-12-10T18:52:17.294Z 24 480-499 45 45 45 75 75 75 F F F
63292023-12-10T18:52:17.294Z 25 500-519 38 38 38 63 63 63 F F F
63302023-12-10T18:52:17.294Z 26 520-539 44 44 44 74 74 74 F F F
63312023-12-10T18:52:17.294Z 27 540-559 42 42 42 69 69 69 F F F
63322023-12-10T18:52:17.294Z 28 560-579 50 50 40 83 83 66 F F T <---
63332023-12-10T18:52:17.295Z 29 580-599 50 50 38 83 83 61 F F T <---
63342023-12-10T18:52:17.295ZMax gen: 50, Max flush: 83
63352023-12-10T18:52:17.295ZError: Difference in extent metadata found!
63362023-12-10T18:52:17.298ZA Difference in extent metadata is expected here
63372023-12-10T18:52:17.298ZOn loop 25
63382023-12-10T18:52:17.298Z
63392023-12-10T18:52:17.298ZVerifying data now
63402023-12-10T18:52:23.590ZNo change after verify
63412023-12-10T18:52:23.590ZLoop: 25 Downstairs dump after verify (and repair):
63422023-12-10T18:52:23.614ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
63432023-12-10T18:52:23.614Z 0 000-019 49 49 49 82 82 82 F F F
63442023-12-10T18:52:23.614Z 1 020-039 51 51 51 84 84 84 F F F
63452023-12-10T18:52:23.615Z 2 040-059 49 49 49 82 82 82 F F F
63462023-12-10T18:52:23.615Z 3 060-079 51 51 51 84 84 84 F F F
63472023-12-10T18:52:23.615Z 4 080-099 51 51 51 84 84 84 F F F
63482023-12-10T18:52:23.621Z 5 100-119 49 49 49 82 82 82 F F F
63492023-12-10T18:52:23.621Z 6 120-139 50 50 50 83 83 83 F F F
63502023-12-10T18:52:23.621Z 7 140-159 50 50 50 83 83 83 F F F
63512023-12-10T18:52:23.621Z 8 160-179 46 46 46 78 78 78 F F F
63522023-12-10T18:52:23.621Z 9 180-199 48 48 48 81 81 81 F F F
63532023-12-10T18:52:23.621Z 10 200-219 50 50 50 83 83 83 F F F
63542023-12-10T18:52:23.621Z 11 220-239 51 51 51 84 84 84 F F F
63552023-12-10T18:52:23.621Z 12 240-259 51 51 51 84 84 84 F F F
63562023-12-10T18:52:23.622Z 13 260-279 46 46 46 77 77 77 F F F
63572023-12-10T18:52:23.622Z 14 280-299 46 46 46 78 78 78 F F F
63582023-12-10T18:52:23.622Z 15 300-319 46 46 46 78 78 78 F F F
63592023-12-10T18:52:23.622Z 16 320-339 51 51 51 84 84 84 F F F
63602023-12-10T18:52:23.622Z 17 340-359 47 47 47 79 79 79 F F F
63612023-12-10T18:52:23.622Z 18 360-379 50 50 50 83 83 83 F F F
63622023-12-10T18:52:23.622Z 19 380-399 50 50 50 83 83 83 F F F
63632023-12-10T18:52:23.622Z 20 400-419 49 49 49 82 82 82 F F F
63642023-12-10T18:52:23.623Z 21 420-439 49 49 49 82 82 82 F F F
63652023-12-10T18:52:23.623Z 22 440-459 46 46 46 77 77 77 F F F
63662023-12-10T18:52:23.623Z 23 460-479 49 49 49 82 82 82 F F F
63672023-12-10T18:52:23.623Z 24 480-499 45 45 45 75 75 75 F F F
63682023-12-10T18:52:23.623Z 25 500-519 38 38 38 63 63 63 F F F
63692023-12-10T18:52:23.623Z 26 520-539 44 44 44 74 74 74 F F F
63702023-12-10T18:52:23.623Z 27 540-559 42 42 42 69 69 69 F F F
63712023-12-10T18:52:23.624Z 28 560-579 50 50 50 83 83 83 F F F
63722023-12-10T18:52:23.624Z 29 580-599 50 50 50 83 83 83 F F F
63732023-12-10T18:52:23.624ZMax gen: 51, Max flush: 84
63742023-12-10T18:52:23.624Z
63752023-12-10T18:52:23.624ZBegin loop 26
63762023-12-10T18:52:23.624ZDownstairs to restart: 0
63772023-12-10T18:52:23.654Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.653600148Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370}
63782023-12-10T18:52:23.654Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.654033432Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370}
63792023-12-10T18:52:23.654Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.654062292Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370}
63802023-12-10T18:52:23.654Z{"msg":"Crucible c940ee80-0fa7-49c1-a561-f3fe22f28f56 has session id: b7314389-5fb7-42b9-8d98-c607a9faf695","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.654083394Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
63812023-12-10T18:52:23.665Z{"msg":"Upstairs opts: Upstairs UUID: c940ee80-0fa7-49c1-a561-f3fe22f28f56, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.654192063Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
63822023-12-10T18:52:23.665Z{"msg":"Crucible stats registered with UUID: c940ee80-0fa7-49c1-a561-f3fe22f28f56","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.654223189Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
63832023-12-10T18:52:23.665ZCrucible runtime is spawned
63842023-12-10T18:52:23.665ZThe guest has requested activation with gen:52
63852023-12-10T18:52:23.665Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.654488647Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695","task":"up_listen"}
63862023-12-10T18:52:23.666Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.654571882Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"looper":"2","session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
63872023-12-10T18:52:23.666Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.654652019Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
63882023-12-10T18:52:23.666Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.654694301Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
63892023-12-10T18:52:23.666Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.654911833Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"looper":"1","session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
63902023-12-10T18:52:23.666Z{"msg":"[2] c940ee80-0fa7-49c1-a561-f3fe22f28f56 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.654965252Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"looper":"2","session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
63912023-12-10T18:52:23.666Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.655145632Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"looper":"0","session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
63922023-12-10T18:52:23.666Z{"msg":"[1] c940ee80-0fa7-49c1-a561-f3fe22f28f56 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.65522972Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"looper":"1","session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
63932023-12-10T18:52:23.667Z{"msg":"[0] c940ee80-0fa7-49c1-a561-f3fe22f28f56 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.655289753Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"looper":"0","session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
63942023-12-10T18:52:23.667Z{"msg":"c940ee80-0fa7-49c1-a561-f3fe22f28f56 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.655786524Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
63952023-12-10T18:52:23.667Z{"msg":"Set desired generation to :52","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.65586936Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
63962023-12-10T18:52:23.667Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.655983841Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
63972023-12-10T18:52:23.668Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.656284839Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
63982023-12-10T18:52:23.668Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.656361478Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
63992023-12-10T18:52:23.668Z{"msg":"[2] c940ee80-0fa7-49c1-a561-f3fe22f28f56 (b7314389-5fb7-42b9-8d98-c607a9faf695) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.65664195Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64002023-12-10T18:52:23.668Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.656704525Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64012023-12-10T18:52:23.668Z{"msg":"[2] client is_active_req TRUE, promote! session b7314389-5fb7-42b9-8d98-c607a9faf695","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.656730612Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64022023-12-10T18:52:23.668Z{"msg":"[1] c940ee80-0fa7-49c1-a561-f3fe22f28f56 (b7314389-5fb7-42b9-8d98-c607a9faf695) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.65688217Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64032023-12-10T18:52:23.668Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.656959013Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64042023-12-10T18:52:23.668Z{"msg":"[1] client is_active_req TRUE, promote! session b7314389-5fb7-42b9-8d98-c607a9faf695","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.65700136Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64052023-12-10T18:52:23.668Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.65744922Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64062023-12-10T18:52:23.669Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.657539372Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64072023-12-10T18:52:23.669Z{"msg":"c940ee80-0fa7-49c1-a561-f3fe22f28f56 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.657679764Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64082023-12-10T18:52:23.669Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.657724952Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64092023-12-10T18:52:23.669Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.65780225Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64102023-12-10T18:52:23.669Z{"msg":"c940ee80-0fa7-49c1-a561-f3fe22f28f56 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.657845954Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64112023-12-10T18:52:23.669Z{"msg":"[2] c940ee80-0fa7-49c1-a561-f3fe22f28f56 (b7314389-5fb7-42b9-8d98-c607a9faf695) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.65836802Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64122023-12-10T18:52:23.669Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.658480471Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64132023-12-10T18:52:23.669Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:23.658515014Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64142023-12-10T18:52:23.669Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.658562072Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64152023-12-10T18:52:23.670Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.658586772Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64162023-12-10T18:52:23.670Z{"msg":"c940ee80-0fa7-49c1-a561-f3fe22f28f56 New WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.658608013Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64172023-12-10T18:52:23.670Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.658628663Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64182023-12-10T18:52:23.670Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.658648585Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64192023-12-10T18:52:23.670Z{"msg":"[0] c940ee80-0fa7-49c1-a561-f3fe22f28f56 (b7314389-5fb7-42b9-8d98-c607a9faf695) New WaitActive WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.658671781Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64202023-12-10T18:52:23.670Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.658722344Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64212023-12-10T18:52:23.670Z{"msg":"[0] client is_active_req TRUE, promote! session b7314389-5fb7-42b9-8d98-c607a9faf695","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.658746253Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64222023-12-10T18:52:23.671Z{"msg":"[1] c940ee80-0fa7-49c1-a561-f3fe22f28f56 (b7314389-5fb7-42b9-8d98-c607a9faf695) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.658842588Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64232023-12-10T18:52:23.671Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.658895157Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64242023-12-10T18:52:23.671Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:23.658919302Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64252023-12-10T18:52:23.671Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.658939622Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64262023-12-10T18:52:23.671Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.658982702Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64272023-12-10T18:52:23.671Z{"msg":"c940ee80-0fa7-49c1-a561-f3fe22f28f56 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.659006176Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64282023-12-10T18:52:23.671Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.659029155Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64292023-12-10T18:52:23.672Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.659073975Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64302023-12-10T18:52:23.672Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.659470482Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64312023-12-10T18:52:23.672Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.65952821Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64322023-12-10T18:52:23.672Z{"msg":"c940ee80-0fa7-49c1-a561-f3fe22f28f56 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.659553428Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64332023-12-10T18:52:23.680Z{"msg":"[0] c940ee80-0fa7-49c1-a561-f3fe22f28f56 (b7314389-5fb7-42b9-8d98-c607a9faf695) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.660125704Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64342023-12-10T18:52:23.680Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.660193344Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64352023-12-10T18:52:23.680Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:23.660218517Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64362023-12-10T18:52:23.680Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.660238545Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64372023-12-10T18:52:23.680Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.6602589Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64382023-12-10T18:52:23.681Z{"msg":"c940ee80-0fa7-49c1-a561-f3fe22f28f56 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.660301071Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64392023-12-10T18:52:23.681Z{"msg":"[0]R flush_numbers[0..12]: [82, 84, 82, 84, 84, 82, 83, 83, 78, 81, 83, 84]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.660325182Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64402023-12-10T18:52:23.681Z{"msg":"[0]R generation[0..12]: [49, 51, 49, 51, 51, 49, 50, 50, 46, 48, 50, 51]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.660366749Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64412023-12-10T18:52:23.681Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.660391414Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64422023-12-10T18:52:23.681Z{"msg":"[1]R flush_numbers[0..12]: [82, 84, 82, 84, 84, 82, 83, 83, 78, 81, 83, 84]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.660414496Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64432023-12-10T18:52:23.681Z{"msg":"[1]R generation[0..12]: [49, 51, 49, 51, 51, 49, 50, 50, 46, 48, 50, 51]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:52
64442023-12-10T18:52:23.681Z,"time":"2023-12-10T18:52:23.660435683Z","hostname":"Wait for a query_work_queue command to finish before sending IO
64452023-12-10T18:52:23.681Zip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64462023-12-10T18:52:23.681Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.660483184Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64472023-12-10T18:52:23.682Z{"msg":"[2]R flush_numbers[0..12]: [82, 84, 82, 84, 84, 82, 83, 83, 78, 81, 83, 84]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.66050683Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64482023-12-10T18:52:23.682Z{"msg":"[2]R generation[0..12]: [49, 51, 49, 51, 51, 49, 50, 50, 46, 48, 50, 51]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.660559668Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64492023-12-10T18:52:23.682Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.660583566Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64502023-12-10T18:52:23.682Z{"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.660640628Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64512023-12-10T18:52:23.682Z{"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.660681121Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64522023-12-10T18:52:23.682Z{"msg":"Next flush: 85","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.660746026Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64532023-12-10T18:52:23.683Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.660851789Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64542023-12-10T18:52:23.683Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.660892597Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64552023-12-10T18:52:23.683Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.660949917Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64562023-12-10T18:52:23.683Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.660974877Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64572023-12-10T18:52:23.683Z{"msg":"c940ee80-0fa7-49c1-a561-f3fe22f28f56 is now active with session: b7314389-5fb7-42b9-8d98-c607a9faf695","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.660996605Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64582023-12-10T18:52:23.683Z{"msg":"c940ee80-0fa7-49c1-a561-f3fe22f28f56 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.661017359Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64592023-12-10T18:52:23.683Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.661038368Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64602023-12-10T18:52:23.684Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.661094358Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64612023-12-10T18:52:23.684Z{"msg":"[0] received reconcile message","v":0,"name":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
64622023-12-10T18:52:23.684Z"crucible","level":30,"time":"2023-12-10T18:52:23.66114446Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64632023-12-10T18:52:23.684Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.661193816Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64642023-12-10T18:52:23.684Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.661217487Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64652023-12-10T18:52:23.684Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.661237446Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64662023-12-10T18:52:23.684Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.661277878Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64672023-12-10T18:52:23.684Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.661299898Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64682023-12-10T18:52:23.684Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.661319684Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64692023-12-10T18:52:23.685Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.661359695Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64702023-12-10T18:52:23.685Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:23.661381568Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1370,"session_id":"b7314389-5fb7-42b9-8d98-c607a9faf695"}
64712023-12-10T18:52:23.685ZLoading write count information from file "/tmp/test_repair_verify.data"
64722023-12-10T18:52:23.685ZRun Repair workload
64732023-12-10T18:52:23.685Z01/30 Write block 494 len 4096 data: 3
64742023-12-10T18:52:23.685Z02/30 Read block 559 len 8192
64752023-12-10T18:52:23.685Z03/30 Read block 381 len 24576
64762023-12-10T18:52:23.685Z04/30 Write block 293 len 16384 data: 4 4 3 5
64772023-12-10T18:52:23.686Z05/30 Read block 570 len 28672
64782023-12-10T18:52:23.686Z06/30 Read block 361 len 12288
64792023-12-10T18:52:23.686Z07/30 Write block 123 len 36864 data: 9 10 8 8 7 7 7 5 4
64802023-12-10T18:52:23.687Z08/30 Read block 213 len 8192
64812023-12-10T18:52:23.687Z09/30 Read block 451 len 12288
64822023-12-10T18:52:23.687Z10/30 Write block 488 len 16384 data: 6 6 6 7
64832023-12-10T18:52:23.692Z11/30 Write block 32 len 24576 data: 4 3 3 4 5 5
64842023-12-10T18:52:23.692Z12/30 Read block 344 len 28672
64852023-12-10T18:52:23.692Z13/30 Read block 489 len 32768
64862023-12-10T18:52:23.692Z14/30 Flush
64872023-12-10T18:52:23.693Z15/30 Write block 204 len 40960 data: 5 6 4 4 3 2 2 3 4 5
64882023-12-10T18:52:23.693Z16/30 Read block 246 len 20480
64892023-12-10T18:52:23.693Z17/30 Read block 134 len 8192
64902023-12-10T18:52:23.693Z18/30 Write block 217 len 32768 data: 10 8 7 7 9 9 7 7
64912023-12-10T18:52:23.699Z19/30 Write block 531 len 28672 data: 6 6 5 6 7 4 4
64922023-12-10T18:52:23.699Z20/30 Write block 324 len 40960 data: 5 4 5 5 5 4 4 3 4 4
64932023-12-10T18:52:23.699Z21/30 Read block 553 len 24576
64942023-12-10T18:52:23.705Z22/30 Read block 565 len 20480
64952023-12-10T18:52:23.705Z23/30 Write block 503 len 4096 data: 6
64962023-12-10T18:52:23.705Z24/30 Flush
64972023-12-10T18:52:23.709Z25/30 Write block 26 len 28672 data: 4 2 2 3 3 4 5
64982023-12-10T18:52:23.709Z26/30 Read block 46 len 36864
64992023-12-10T18:52:23.710Z27/30 Read block 14 len 36864
65002023-12-10T18:52:23.714Z28/30 Write block 386 len 28672 data: 6 7 6 5 6 4 5
65012023-12-10T18:52:23.715Z29/30 Read block 218 len 28672
65022023-12-10T18:52:23.722Z30/30 Write block 332 len 32768 data: 5 5 3 2 2 2 2 2
65032023-12-10T18:52:23.722Z----------------------------------------------------------------
65042023-12-10T18:52:23.723Z Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30
65052023-12-10T18:52:23.723ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
65062023-12-10T18:52:23.723Z 1 Acked 1000 Write 1 Done Done Done false
65072023-12-10T18:52:23.723Z 2 Acked 1001 Read 2 Done Done Done false
65082023-12-10T18:52:23.723Z 3 Acked 1002 Read 6 Sent Done Done false
65092023-12-10T18:52:23.724Z 4 Acked 1003 Write 4 Sent Done Done false
65102023-12-10T18:52:23.724Z 5 Acked 1004 Read 7 Sent Done Done false
65112023-12-10T18:52:23.724Z 6 Acked 1005 Read 3 Sent Done Done false
65122023-12-10T18:52:23.724Z 7 Acked 1006 Write 9 Sent Done Done false
65132023-12-10T18:52:23.724Z 8 Acked 1007 Read 2 Sent Done Done false
65142023-12-10T18:52:23.724Z 9 Acked 1008 Read 3 Sent Done Done false
65152023-12-10T18:52:23.724Z 10 Acked 1009 Write 4 Sent Done Done false
65162023-12-10T18:52:23.724Z 11 Acked 1010 Write 6 Sent Done Done false
65172023-12-10T18:52:23.725Z 12 Acked 1011 Read 7 Sent Done Done false
65182023-12-10T18:52:23.725Z 13 Acked 1012 Read 8 Sent Done Done false
65192023-12-10T18:52:23.725Z 14 Acked 1013 Flush 0 Sent Done Done false
65202023-12-10T18:52:23.725Z 15 Acked 1014 Write 10 Sent Done Done false
65212023-12-10T18:52:23.725Z 16 Acked 1015 Read 5 Sent Done Done false
65222023-12-10T18:52:23.725Z 17 Acked 1016 Read 2 Sent Done Done false
65232023-12-10T18:52:23.725Z 18 Acked 1017 Write 8 Sent Done Done false
65242023-12-10T18:52:23.725Z 19 Acked 1018 Write 7 Sent Done Done false
65252023-12-10T18:52:23.725Z 20 Acked 1019 Write 10 Sent Done Done false
65262023-12-10T18:52:23.726Z 21 Acked 1020 Read 6 Sent Done Done false
65272023-12-10T18:52:23.726Z 22 Acked 1021 Read 5 Sent Done Done false
65282023-12-10T18:52:23.726Z 23 Acked 1022 Write 1 Sent Done Done false
65292023-12-10T18:52:23.726Z 24 Acked 1023 Flush 0 Sent Done Done false
65302023-12-10T18:52:23.726Z 25 Acked 1024 Write 7 Sent Done Done false
65312023-12-10T18:52:23.726Z 26 Acked 1025 Read 9 Sent Done Done false
65322023-12-10T18:52:23.726Z 27 Acked 1026 Read 9 Sent Done Done false
65332023-12-10T18:52:23.726Z 28 Acked 1027 Write 7 Sent Done Done false
65342023-12-10T18:52:23.727Z 29 Acked 1028 Read 7 Sent Done Done false
65352023-12-10T18:52:23.727Z 30 Acked 1029 Write 8 Sent Sent Sent false
65362023-12-10T18:52:23.727Z STATES DS:0 DS:1 DS:2 TOTAL
65372023-12-10T18:52:23.727Z New 0 0 0 0
65382023-12-10T18:52:23.727Z Sent 28 1 1 30
65392023-12-10T18:52:23.727Z Done 2 29 29 60
65402023-12-10T18:52:23.727Z Skipped 0 0 0 0
65412023-12-10T18:52:23.727Z Error 0 0 0 0
65422023-12-10T18:52:23.727ZLast Flush: 0 1023 1023
65432023-12-10T18:52:23.728ZDownstairs last five completed:
65442023-12-10T18:52:23.728ZUpstairs last five completed: 30 29 28 27 26 25
65452023-12-10T18:52:23.728ZWrote out file "/tmp/test_repair_verify.data"
65462023-12-10T18:52:23.740Z
65472023-12-10T18:52:23.740ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
65482023-12-10T18:52:23.759ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
65492023-12-10T18:52:23.759Z 0 000-019 49 49 49 82 82 82 F F F
65502023-12-10T18:52:23.765Z 1 020-039 51 52 52 84 85 85 F T T <---
65512023-12-10T18:52:23.765Z 2 040-059 49 49 49 82 82 82 F F F
65522023-12-10T18:52:23.765Z 3 060-079 51 51 51 84 84 84 F F F
65532023-12-10T18:52:23.765Z 4 080-099 51 51 51 84 84 84 F F F
65542023-12-10T18:52:23.765Z 5 100-119 49 49 49 82 82 82 F F F
65552023-12-10T18:52:23.766Z 6 120-139 50 52 52 83 85 85 F F F <---
65562023-12-10T18:52:23.766Z 7 140-159 50 50 50 83 83 83 F F F
65572023-12-10T18:52:23.766Z 8 160-179 46 46 46 78 78 78 F F F
65582023-12-10T18:52:23.766Z 9 180-199 48 48 48 81 81 81 F F F
65592023-12-10T18:52:23.766Z 10 200-219 50 52 52 83 86 86 F F F <---
65602023-12-10T18:52:23.766Z 11 220-239 51 52 52 84 86 86 F F F <---
65612023-12-10T18:52:23.766Z 12 240-259 51 51 51 84 84 84 F F F
65622023-12-10T18:52:23.766Z 13 260-279 46 46 46 77 77 77 F F F
65632023-12-10T18:52:23.767Z 14 280-299 46 52 52 78 85 85 F F F <---
65642023-12-10T18:52:23.767Z 15 300-319 46 46 46 78 78 78 F F F
65652023-12-10T18:52:23.767Z 16 320-339 51 52 52 84 86 86 F T T <---
65662023-12-10T18:52:23.767Z 17 340-359 47 47 47 79 79 79 F F F
65672023-12-10T18:52:23.767Z 18 360-379 50 50 50 83 83 83 F F F
65682023-12-10T18:52:23.767Z 19 380-399 50 50 50 83 83 83 F T T <---
65692023-12-10T18:52:23.767Z 20 400-419 49 49 49 82 82 82 F F F
65702023-12-10T18:52:23.767Z 21 420-439 49 49 49 82 82 82 F F F
65712023-12-10T18:52:23.767Z 22 440-459 46 46 46 77 77 77 F F F
65722023-12-10T18:52:23.768Z 23 460-479 49 49 49 82 82 82 F F F
65732023-12-10T18:52:23.768Z 24 480-499 45 52 52 75 85 85 T F F <---
65742023-12-10T18:52:23.768Z 25 500-519 38 52 52 63 86 86 F F F <---
65752023-12-10T18:52:23.768Z 26 520-539 44 52 52 74 86 86 F F F <---
65762023-12-10T18:52:23.768Z 27 540-559 42 42 42 69 69 69 F F F
65772023-12-10T18:52:23.769Z 28 560-579 50 50 50 83 83 83 F F F
65782023-12-10T18:52:23.769Z 29 580-599 50 50 50 83 83 83 F F F
65792023-12-10T18:52:23.769ZMax gen: 52, Max flush: 86
65802023-12-10T18:52:23.770ZError: Difference in extent metadata found!
65812023-12-10T18:52:23.777ZA Difference in extent metadata is expected here
65822023-12-10T18:52:23.777ZOn loop 26
65832023-12-10T18:52:23.778Z
65842023-12-10T18:52:23.778ZVerifying data now
65852023-12-10T18:52:28.821ZNo change after verify
65862023-12-10T18:52:28.821ZLoop: 26 Downstairs dump after verify (and repair):
65872023-12-10T18:52:28.855ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
65882023-12-10T18:52:28.855Z 0 000-019 49 49 49 82 82 82 F F F
65892023-12-10T18:52:28.855Z 1 020-039 53 53 53 87 87 87 F F F
65902023-12-10T18:52:28.855Z 2 040-059 49 49 49 82 82 82 F F F
65912023-12-10T18:52:28.855Z 3 060-079 51 51 51 84 84 84 F F F
65922023-12-10T18:52:28.855Z 4 080-099 51 51 51 84 84 84 F F F
65932023-12-10T18:52:28.856Z 5 100-119 49 49 49 82 82 82 F F F
65942023-12-10T18:52:28.856Z 6 120-139 52 52 52 85 85 85 F F F
65952023-12-10T18:52:28.856Z 7 140-159 50 50 50 83 83 83 F F F
65962023-12-10T18:52:28.856Z 8 160-179 46 46 46 78 78 78 F F F
65972023-12-10T18:52:28.871Z 9 180-199 48 48 48 81 81 81 F F F
65982023-12-10T18:52:28.872Z 10 200-219 52 52 52 86 86 86 F F F
65992023-12-10T18:52:28.872Z 11 220-239 52 52 52 86 86 86 F F F
66002023-12-10T18:52:28.872Z 12 240-259 51 51 51 84 84 84 F F F
66012023-12-10T18:52:28.872Z 13 260-279 46 46 46 77 77 77 F F F
66022023-12-10T18:52:28.872Z 14 280-299 52 52 52 85 85 85 F F F
66032023-12-10T18:52:28.872Z 15 300-319 46 46 46 78 78 78 F F F
66042023-12-10T18:52:28.872Z 16 320-339 53 53 53 87 87 87 F F F
66052023-12-10T18:52:28.872Z 17 340-359 47 47 47 79 79 79 F F F
66062023-12-10T18:52:28.872Z 18 360-379 50 50 50 83 83 83 F F F
66072023-12-10T18:52:28.872Z 19 380-399 53 53 53 87 87 87 F F F
66082023-12-10T18:52:28.873Z 20 400-419 49 49 49 82 82 82 F F F
66092023-12-10T18:52:28.873Z 21 420-439 49 49 49 82 82 82 F F F
66102023-12-10T18:52:28.874Z 22 440-459 46 46 46 77 77 77 F F F
66112023-12-10T18:52:28.874Z 23 460-479 49 49 49 82 82 82 F F F
66122023-12-10T18:52:28.874Z 24 480-499 52 52 52 85 85 85 F F F
66132023-12-10T18:52:28.874Z 25 500-519 52 52 52 86 86 86 F F F
66142023-12-10T18:52:28.874Z 26 520-539 52 52 52 86 86 86 F F F
66152023-12-10T18:52:28.874Z 27 540-559 42 42 42 69 69 69 F F F
66162023-12-10T18:52:28.874Z 28 560-579 50 50 50 83 83 83 F F F
66172023-12-10T18:52:28.874Z 29 580-599 50 50 50 83 83 83 F F F
66182023-12-10T18:52:28.874ZMax gen: 53, Max flush: 87
66192023-12-10T18:52:28.874Z
66202023-12-10T18:52:28.874ZBegin loop 27
66212023-12-10T18:52:28.875ZDownstairs to restart: 2
66222023-12-10T18:52:28.895Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.875202942Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378}
66232023-12-10T18:52:28.895Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.875609246Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378}
66242023-12-10T18:52:28.895Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.875638627Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378}
66252023-12-10T18:52:28.895Z{"msg":"Crucible eb07baca-3acc-4158-971c-37f853362f64 has session id: 9a735997-a929-4efa-b896-5b1c17ab468c","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.875683603Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66262023-12-10T18:52:28.896Z{"msg":"Upstairs opts: Upstairs UUID: eb07baca-3acc-4158-971c-37f853362f64, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.875722411Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66272023-12-10T18:52:28.896Z{Crucible runtime is spawned
66282023-12-10T18:52:28.896Z"msg":"Crucible stats registered with UUID: eb07baca-3acc-4158-971c-37f853362f64","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.875800993Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66292023-12-10T18:52:28.896ZThe guest has requested activation with gen:54
66302023-12-10T18:52:28.896Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.876028234Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"looper":"0","session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66312023-12-10T18:52:28.897Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.876111604Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c","task":"up_listen"}
66322023-12-10T18:52:28.897Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.876154456Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"looper":"1","session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66332023-12-10T18:52:28.897Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.876220786Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"looper":"2","session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66342023-12-10T18:52:28.897Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.876311414Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66352023-12-10T18:52:28.897Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.876367178Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66362023-12-10T18:52:28.898Z{"msg":"eb07baca-3acc-4158-971c-37f853362f64 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.876435429Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66372023-12-10T18:52:28.898Z{"msg":"[1] eb07baca-3acc-4158-971c-37f853362f64 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.876476624Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"looper":"1","session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66382023-12-10T18:52:28.898Z{"msg":"Set desired generation to :54","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.876568453Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66392023-12-10T18:52:28.898Z{"msg":"[0] eb07baca-3acc-4158-971c-37f853362f64 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.876615531Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"looper":"0","session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66402023-12-10T18:52:28.898Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.876655225Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66412023-12-10T18:52:28.898Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.877036301Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66422023-12-10T18:52:28.899Z{"msg":"[0] eb07baca-3acc-4158-971c-37f853362f64 (9a735997-a929-4efa-b896-5b1c17ab468c) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.877390116Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66432023-12-10T18:52:28.899Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.87746562Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66442023-12-10T18:52:28.899Z{"msg":"[0] client is_active_req TRUE, promote! session 9a735997-a929-4efa-b896-5b1c17ab468c","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.87751624Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66452023-12-10T18:52:28.899Z{"msg":"[1] eb07baca-3acc-4158-971c-37f853362f64 (9a735997-a929-4efa-b896-5b1c17ab468c) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.877594336Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66462023-12-10T18:52:28.899Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.877638228Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66472023-12-10T18:52:28.900Z{"msg":"[1] client is_active_req TRUE, promote! session 9a735997-a929-4efa-b896-5b1c17ab468c","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.87771471Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66482023-12-10T18:52:28.900Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.878292637Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66492023-12-10T18:52:28.900Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.878372541Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66502023-12-10T18:52:28.900Z{"msg":"eb07baca-3acc-4158-971c-37f853362f64 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.878416023Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66512023-12-10T18:52:28.900Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.878614303Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66522023-12-10T18:52:28.900Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.87870098Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66532023-12-10T18:52:28.901Z{"msg":"eb07baca-3acc-4158-971c-37f853362f64 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.878743418Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66542023-12-10T18:52:28.901Z{"msg":"[0] eb07baca-3acc-4158-971c-37f853362f64 (9a735997-a929-4efa-b896-5b1c17ab468c) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.879132149Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66552023-12-10T18:52:28.901Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.879217474Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66562023-12-10T18:52:28.902Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:28.879292029Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66572023-12-10T18:52:28.902Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.87946287Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66582023-12-10T18:52:28.902Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.879531953Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66592023-12-10T18:52:28.902Z{"msg":"eb07baca-3acc-4158-971c-37f853362f64 WaitQuorum WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.879574304Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66602023-12-10T18:52:28.902Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.879631499Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66612023-12-10T18:52:28.902Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.879670278Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66622023-12-10T18:52:28.903Z{"msg":"[1] eb07baca-3acc-4158-971c-37f853362f64 (9a735997-a929-4efa-b896-5b1c17ab468c) WaitQuorum WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.879728738Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66632023-12-10T18:52:28.903Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.879767907Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66642023-12-10T18:52:28.903Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:28.879820875Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66652023-12-10T18:52:28.903Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.879859002Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66662023-12-10T18:52:28.903Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.879914079Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66672023-12-10T18:52:28.904Z{"msg":"eb07baca-3acc-4158-971c-37f853362f64 WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.87995387Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66682023-12-10T18:52:28.904Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.880007281Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66692023-12-10T18:52:28.904Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:28.880045398Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66702023-12-10T18:52:29.878Z{"msg":"[2] eb07baca-3acc-4158-971c-37f853362f64 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.87752964Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"looper":"2","session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66712023-12-10T18:52:29.878Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.877618719Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66722023-12-10T18:52:29.890Z{"msg":"[2] eb07baca-3acc-4158-971c-37f853362f64 (9a735997-a929-4efa-b896-5b1c17ab468c) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.878627763Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66732023-12-10T18:52:29.890Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.878685461Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66742023-12-10T18:52:29.891Z{"msg":"[2] client is_active_req TRUE, promote! session 9a735997-a929-4efa-b896-5b1c17ab468c","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.878711589Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66752023-12-10T18:52:29.891Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.879333488Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66762023-12-10T18:52:29.891Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.879443798Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66772023-12-10T18:52:29.891Z{"msg":"eb07baca-3acc-4158-971c-37f853362f64 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.879473887Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66782023-12-10T18:52:29.892Z{"msg":"[2] eb07baca-3acc-4158-971c-37f853362f64 (9a735997-a929-4efa-b896-5b1c17ab468c) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.880083411Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66792023-12-10T18:52:29.892Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.880136794Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66802023-12-10T18:52:29.892Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:29.880160788Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66812023-12-10T18:52:29.892Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.880181152Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66822023-12-10T18:52:29.892Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.880201978Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66832023-12-10T18:52:29.893Z{"msg":"eb07baca-3acc-4158-971c-37f853362f64 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.880243498Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66842023-12-10T18:52:29.895Z{"msg":"[0]R flush_numbers[0..12]: [82, 87, 82, 84, 84, 82, 85, 83, 78, 81, 86, 86]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.88026778Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66852023-12-10T18:52:29.895Z{"msg":"[0]R generation[0..12]: [49, 53, 49, 51, 51, 49, 52, 50, 46, 48, 52, 52]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.880289944Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66862023-12-10T18:52:29.895Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.880332075Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66872023-12-10T18:52:29.895Z{"msg":"[1]R flush_numbers[0..12]: [82, 87, 82, 84, 84, 82, 85, 83, 78, 81, 86, 86]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.880355775Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66882023-12-10T18:52:29.896Z{"msg":"[1]R generation[0..12]: [49, 53, 49, 51, 51, 49, 52, 50, 46, 48, 52, 52]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.880397121Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66892023-12-10T18:52:29.896Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.880421545Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66902023-12-10T18:52:29.896Z{"msg":"[2]R flush_numbers[0..12]: [82, 87, 82, 84, 84, 82, 85, 83, 78, 81, 86, 86]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.880442209Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66912023-12-10T18:52:29.896Z{"msg":"[2]R generation[0..12]: [49, 53, 49, 51, 51, 49, 52, 50, 46, 48, 52, 52]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.880462855Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66922023-12-10T18:52:29.897ZThe guest has finished waiting for activation with:54
66932023-12-10T18:52:29.897Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]Wait for a query_work_queue command to finish before sending IO
66942023-12-10T18:52:29.897Z","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.880512992Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66952023-12-10T18:52:29.900Z{"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.880536192Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66962023-12-10T18:52:29.900Z{"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.880576629Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66972023-12-10T18:52:29.900Z{"msg":"Next flush: 88","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.880598771Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66982023-12-10T18:52:29.901Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.880643685Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
66992023-12-10T18:52:29.901Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.880665864Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
67002023-12-10T18:52:29.901Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.880732129Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
67012023-12-10T18:52:29.901Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.880768664Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
67022023-12-10T18:52:29.901Z{"msg":"eb07baca-3acc-4158-971c-37f853362f64 is now active with session: 9a735997-a929-4efa-b896-5b1c17ab468c","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.880853207Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
67032023-12-10T18:52:29.902Z{"msg":"eb07baca-3acc-4158-971c-37f853362f64 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.880879246Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
67042023-12-10T18:52:29.902Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.880899828Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
67052023-12-10T18:52:29.902Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.880968414Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
67062023-12-10T18:52:29.903Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.881043308Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
67072023-12-10T18:52:29.903Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.881084366Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
67082023-12-10T18:52:29.903Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.881157318Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
67092023-12-10T18:52:29.903Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.881229646Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
67102023-12-10T18:52:29.904Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.881303619Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
67112023-12-10T18:52:29.904Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.881343986Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
67122023-12-10T18:52:29.904Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.881414395Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
67132023-12-10T18:52:29.904Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.881456751Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
67142023-12-10T18:52:29.905Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:29.881525101Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1378,"session_id":"9a735997-a929-4efa-b896-5b1c17ab468c"}
67152023-12-10T18:52:29.905ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
67162023-12-10T18:52:29.905ZLoading write count information from file "/tmp/test_repair_verify.data"
67172023-12-10T18:52:29.905ZRun Repair workload
67182023-12-10T18:52:29.906Z01/30 Write block 213 len 28672 data: 6 8 9 10 11 9 8
67192023-12-10T18:52:29.917Z02/30 Read block 4 len 28672
67202023-12-10T18:52:29.917Z03/30 Read block 522 len 40960
67212023-12-10T18:52:29.917Z04/30 Read block 117 len 4096
67222023-12-10T18:52:29.918Z05/30 Write block 324 len 40960 data: 6 5 6 6 6 5 5 4 6 6
67232023-12-10T18:52:29.918Z06/30 Write block 121 len 20480 data: 8 8 10 11 9
67242023-12-10T18:52:29.918Z07/30 Read block 205 len 4096
67252023-12-10T18:52:29.918Z08/30 Write block 314 len 40960 data: 4 3 3 3 3 4 4 4 6 5
67262023-12-10T18:52:29.918Z09/30 Read block 425 len 8192
67272023-12-10T18:52:29.919Z10/30 Read block 295 len 32768
67282023-12-10T18:52:29.919Z11/30 Write block 371 len 12288 data: 4 5 3
67292023-12-10T18:52:29.919Z12/30 Read block 498 len 16384
67302023-12-10T18:52:29.919Z13/30 Write block 64 len 4096 data: 4
67312023-12-10T18:52:29.920Z14/30 Read block 71 len 20480
67322023-12-10T18:52:29.920Z15/30 Read block 591 len 4096
67332023-12-10T18:52:29.920Z16/30 Read block 236 len 36864
67342023-12-10T18:52:29.920Z17/30 Read block 268 len 24576
67352023-12-10T18:52:29.920Z18/30 Flush
67362023-12-10T18:52:29.927Z19/30 Flush
67372023-12-10T18:52:29.928Z20/30 Write block 342 len 12288 data: 4 5 6
67382023-12-10T18:52:29.928Z21/30 Read block 574 len 28672
67392023-12-10T18:52:29.928Z22/30 Write block 171 len 16384 data: 6 6 6 6
67402023-12-10T18:52:29.928Z23/30 Flush
67412023-12-10T18:52:29.928Z24/30 Write block 138 len 24576 data: 3 3 2 2 3 3
67422023-12-10T18:52:29.928Z25/30 Write block 462 len 16384 data: 4 4 4 6
67432023-12-10T18:52:29.928Z26/30 Flush
67442023-12-10T18:52:29.939Z27/30 Write block 264 len 12288 data: 4 4 3
67452023-12-10T18:52:29.940Z28/30 Read block 277 len 32768
67462023-12-10T18:52:29.940Z29/30 Read block 292 len 12288
67472023-12-10T18:52:29.940Z30/30 Read block 42 len 32768
67482023-12-10T18:52:29.949Z----------------------------------------------------------------
67492023-12-10T18:52:29.949Z Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30
67502023-12-10T18:52:29.949ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
67512023-12-10T18:52:29.949Z 1 Acked 1000 Write 7 Done Done Done false
67522023-12-10T18:52:29.949Z 2 Acked 1001 Read 7 Done Done Done false
67532023-12-10T18:52:29.949Z 3 Acked 1002 Read 10 Done Done Done false
67542023-12-10T18:52:29.949Z 4 Acked 1003 Read 1 Done Done Done false
67552023-12-10T18:52:29.949Z 5 Acked 1004 Write 10 Done Done Done false
67562023-12-10T18:52:29.949Z 6 Acked 1005 Write 5 Done Done Done false
67572023-12-10T18:52:29.950Z 7 Acked 1006 Read 1 Done Done Done false
67582023-12-10T18:52:29.950Z 8 Acked 1007 Write 10 Done Done Sent false
67592023-12-10T18:52:29.950Z 9 Acked 1008 Read 2 Done Done Sent false
67602023-12-10T18:52:29.950Z 10 Acked 1009 Read 8 Done Done Sent false
67612023-12-10T18:52:29.950Z 11 Acked 1010 Write 3 Done Done Sent false
67622023-12-10T18:52:29.950Z 12 Acked 1011 Read 4 Done Done Sent false
67632023-12-10T18:52:29.950Z 13 Acked 1012 Write 1 Done Done Sent false
67642023-12-10T18:52:29.950Z 14 Acked 1013 Read 5 Done Done Sent false
67652023-12-10T18:52:29.950Z 15 Acked 1014 Read 1 Done Done Sent false
67662023-12-10T18:52:29.950Z 16 Acked 1015 Read 9 Done Done Sent false
67672023-12-10T18:52:29.950Z 17 Acked 1016 Read 6 Done Done Sent false
67682023-12-10T18:52:29.950Z 18 Acked 1017 Flush 0 Done Done Sent false
67692023-12-10T18:52:29.953Z 19 Acked 1018 Flush 0 Done Done Sent false
67702023-12-10T18:52:29.953Z 20 Acked 1019 Write 3 Done Done Sent false
67712023-12-10T18:52:29.953Z 21 Acked 1020 Read 7 Done Done Sent false
67722023-12-10T18:52:29.953Z 22 Acked 1021 Write 4 Done Done Sent false
67732023-12-10T18:52:29.953Z 23 Acked 1022 Flush 0 Done Done Sent false
67742023-12-10T18:52:29.953Z 24 Acked 1023 Write 6 Done Done Sent false
67752023-12-10T18:52:29.954Z 25 Acked 1024 Write 4 Done Done Sent false
67762023-12-10T18:52:29.954Z 26 Acked 1025 Flush 0 Done Done Sent false
67772023-12-10T18:52:29.954Z 27 Acked 1026 Write 3 Done Done Sent false
67782023-12-10T18:52:29.954Z 28 Acked 1027 Read 8 Done Done Sent false
67792023-12-10T18:52:29.954Z 29 Acked 1028 Read 3 Done Done Sent false
67802023-12-10T18:52:29.954Z 30 Acked 1029 Read 8 Done Sent Sent false
67812023-12-10T18:52:29.955Z STATES DS:0 DS:1 DS:2 TOTAL
67822023-12-10T18:52:29.955Z New 0 0 0 0
67832023-12-10T18:52:29.955Z Sent 0 1 23 24
67842023-12-10T18:52:29.955Z Done 30 29 7 66
67852023-12-10T18:52:29.955Z Skipped 0 0 0 0
67862023-12-10T18:52:29.955Z Error 0 0 0 0
67872023-12-10T18:52:29.955ZLast Flush: 1025 1025 0
67882023-12-10T18:52:29.955ZDownstairs last five completed:
67892023-12-10T18:52:29.956ZUpstairs last five completed: 30 29 28 27 26 25
67902023-12-10T18:52:29.956ZWrote out file "/tmp/test_repair_verify.data"
67912023-12-10T18:52:29.956Z
67922023-12-10T18:52:29.963ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
67932023-12-10T18:52:29.978ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
67942023-12-10T18:52:29.978Z 0 000-019 49 49 49 82 82 82 F F F
67952023-12-10T18:52:29.978Z 1 020-039 53 53 53 87 87 87 F F F
67962023-12-10T18:52:29.979Z 2 040-059 49 49 49 82 82 82 F F F
67972023-12-10T18:52:29.979Z 3 060-079 54 54 51 88 88 84 F F F <---
67982023-12-10T18:52:29.988Z 4 080-099 51 51 51 84 84 84 F F F
67992023-12-10T18:52:29.988Z 5 100-119 49 49 49 82 82 82 F F F
68002023-12-10T18:52:29.988Z 6 120-139 54 54 52 91 91 85 F F T <---
68012023-12-10T18:52:29.988Z 7 140-159 54 54 50 91 91 83 F F F <---
68022023-12-10T18:52:29.988Z 8 160-179 54 54 46 90 90 78 F F F <---
68032023-12-10T18:52:29.988Z 9 180-199 48 48 48 81 81 81 F F F
68042023-12-10T18:52:29.989Z 10 200-219 54 54 52 88 88 86 F F T <---
68052023-12-10T18:52:29.989Z 11 220-239 52 52 52 86 86 86 F F F
68062023-12-10T18:52:29.989Z 12 240-259 51 51 51 84 84 84 F F F
68072023-12-10T18:52:29.989Z 13 260-279 46 46 46 77 77 77 T T F <---
68082023-12-10T18:52:29.990Z 14 280-299 52 52 52 85 85 85 F F F
68092023-12-10T18:52:29.990Z 15 300-319 54 54 46 88 88 78 F F F <---
68102023-12-10T18:52:29.990Z 16 320-339 54 54 53 88 88 87 F F T <---
68112023-12-10T18:52:29.990Z 17 340-359 54 54 47 90 90 79 F F F <---
68122023-12-10T18:52:29.990Z 18 360-379 54 54 50 88 88 83 F F F <---
68132023-12-10T18:52:29.991Z 19 380-399 53 53 53 87 87 87 F F F
68142023-12-10T18:52:29.991Z 20 400-419 49 49 49 82 82 82 F F F
68152023-12-10T18:52:29.991Z 21 420-439 49 49 49 82 82 82 F F F
68162023-12-10T18:52:29.992Z 22 440-459 46 46 46 77 77 77 F F F
68172023-12-10T18:52:29.992Z 23 460-479 54 54 49 91 91 82 F F F <---
68182023-12-10T18:52:29.992Z 24 480-499 52 52 52 85 85 85 F F F
68192023-12-10T18:52:29.992Z 25 500-519 52 52 52 86 86 86 F F F
68202023-12-10T18:52:29.992Z 26 520-539 52 52 52 86 86 86 F F F
68212023-12-10T18:52:29.993Z 27 540-559 42 42 42 69 69 69 F F F
68222023-12-10T18:52:29.993Z 28 560-579 50 50 50 83 83 83 F F F
68232023-12-10T18:52:29.993Z 29 580-599 50 50 50 83 83 83 F F F
68242023-12-10T18:52:29.993ZMax gen: 54, Max flush: 91
68252023-12-10T18:52:29.993ZError: Difference in extent metadata found!
68262023-12-10T18:52:29.994ZA Difference in extent metadata is expected here
68272023-12-10T18:52:29.994ZOn loop 27
68282023-12-10T18:52:29.994Z
68292023-12-10T18:52:29.994ZVerifying data now
68302023-12-10T18:52:36.094ZNo change after verify
68312023-12-10T18:52:36.094ZLoop: 27 Downstairs dump after verify (and repair):
68322023-12-10T18:52:36.115ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
68332023-12-10T18:52:36.115Z 0 000-019 49 49 49 82 82 82 F F F
68342023-12-10T18:52:36.121Z 1 020-039 53 53 53 87 87 87 F F F
68352023-12-10T18:52:36.121Z 2 040-059 49 49 49 82 82 82 F F F
68362023-12-10T18:52:36.121Z 3 060-079 54 54 54 88 88 88 F F F
68372023-12-10T18:52:36.121Z 4 080-099 51 51 51 84 84 84 F F F
68382023-12-10T18:52:36.121Z 5 100-119 49 49 49 82 82 82 F F F
68392023-12-10T18:52:36.121Z 6 120-139 54 54 54 91 91 91 F F F
68402023-12-10T18:52:36.122Z 7 140-159 54 54 54 91 91 91 F F F
68412023-12-10T18:52:36.122Z 8 160-179 54 54 54 90 90 90 F F F
68422023-12-10T18:52:36.122Z 9 180-199 48 48 48 81 81 81 F F F
68432023-12-10T18:52:36.122Z 10 200-219 54 54 54 88 88 88 F F F
68442023-12-10T18:52:36.122Z 11 220-239 52 52 52 86 86 86 F F F
68452023-12-10T18:52:36.122Z 12 240-259 51 51 51 84 84 84 F F F
68462023-12-10T18:52:36.123Z 13 260-279 55 55 55 92 92 92 F F F
68472023-12-10T18:52:36.123Z 14 280-299 52 52 52 85 85 85 F F F
68482023-12-10T18:52:36.123Z 15 300-319 54 54 54 88 88 88 F F F
68492023-12-10T18:52:36.123Z 16 320-339 54 54 54 88 88 88 F F F
68502023-12-10T18:52:36.123Z 17 340-359 54 54 54 90 90 90 F F F
68512023-12-10T18:52:36.124Z 18 360-379 54 54 54 88 88 88 F F F
68522023-12-10T18:52:36.124Z 19 380-399 53 53 53 87 87 87 F F F
68532023-12-10T18:52:36.124Z 20 400-419 49 49 49 82 82 82 F F F
68542023-12-10T18:52:36.124Z 21 420-439 49 49 49 82 82 82 F F F
68552023-12-10T18:52:36.124Z 22 440-459 46 46 46 77 77 77 F F F
68562023-12-10T18:52:36.124Z 23 460-479 54 54 54 91 91 91 F F F
68572023-12-10T18:52:36.125Z 24 480-499 52 52 52 85 85 85 F F F
68582023-12-10T18:52:36.125Z 25 500-519 52 52 52 86 86 86 F F F
68592023-12-10T18:52:36.125Z 26 520-539 52 52 52 86 86 86 F F F
68602023-12-10T18:52:36.125Z 27 540-559 42 42 42 69 69 69 F F F
68612023-12-10T18:52:36.125Z 28 560-579 50 50 50 83 83 83 F F F
68622023-12-10T18:52:36.125Z 29 580-599 50 50 50 83 83 83 F F F
68632023-12-10T18:52:36.125ZMax gen: 55, Max flush: 92
68642023-12-10T18:52:36.125Z
68652023-12-10T18:52:36.125ZBegin loop 28
68662023-12-10T18:52:36.126ZDownstairs to restart: 1
68672023-12-10T18:52:36.559Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.150507868Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386}
68682023-12-10T18:52:37.017Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.150942346Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386}
68692023-12-10T18:52:37.018Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.151006998Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386}
68702023-12-10T18:52:37.018Z{"msg":"Crucible ec5eeed3-0492-436e-8824-9bfc0df6805d has session id: a1a48d63-a10b-4d7b-91db-760c654e736b","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.151082552Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
68712023-12-10T18:52:37.018Z{"msg":"Upstairs opts: Upstairs UUID: ec5eeed3-0492-436e-8824-9bfc0df6805d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.151178084Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
68722023-12-10T18:52:37.018Z{"msg":"Crucible stats registered with UUID: ec5eeed3-0492-436e-8824-9bfc0df6805d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.151223062Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
68732023-12-10T18:52:37.018Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.151306874Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b","task":"up_listen"}
68742023-12-10T18:52:37.018Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.151352744Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
68752023-12-10T18:52:37.018ZCrucible runtime is spawned
68762023-12-10T18:52:37.018Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.151425583Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
68772023-12-10T18:52:37.018Z{"msg":"The guest has requested activation with gen:56
68782023-12-10T18:52:37.019Z[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.151549597Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"looper":"0","session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
68792023-12-10T18:52:37.019Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.15169613Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"looper":"1","session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
68802023-12-10T18:52:37.019Z{"msg":"[0] ec5eeed3-0492-436e-8824-9bfc0df6805d looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.151822265Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"looper":"0","session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
68812023-12-10T18:52:37.019Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.151929681Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"looper":"2","session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
68822023-12-10T18:52:37.019Z{"msg":"[1] ec5eeed3-0492-436e-8824-9bfc0df6805d looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.152037305Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"looper":"1","session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
68832023-12-10T18:52:37.019Z{"msg":"[2] ec5eeed3-0492-436e-8824-9bfc0df6805d looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.152068574Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"looper":"2","session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
68842023-12-10T18:52:37.019Z{"msg":"ec5eeed3-0492-436e-8824-9bfc0df6805d active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.152472337Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
68852023-12-10T18:52:37.020Z{"msg":"Set desired generation to :56","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.152549014Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
68862023-12-10T18:52:37.020Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.152632078Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
68872023-12-10T18:52:37.020Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.152976511Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
68882023-12-10T18:52:37.020Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.153028242Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
68892023-12-10T18:52:37.020Z{"msg":"[0] ec5eeed3-0492-436e-8824-9bfc0df6805d (a1a48d63-a10b-4d7b-91db-760c654e736b) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.153343507Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
68902023-12-10T18:52:37.020Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.153392959Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
68912023-12-10T18:52:37.020Z{"msg":"[0] client is_active_req TRUE, promote! session a1a48d63-a10b-4d7b-91db-760c654e736b","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.153431638Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
68922023-12-10T18:52:37.020Z{"msg":"[2] ec5eeed3-0492-436e-8824-9bfc0df6805d (a1a48d63-a10b-4d7b-91db-760c654e736b) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.153671471Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
68932023-12-10T18:52:37.021Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.153787802Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
68942023-12-10T18:52:37.021Z{"msg":"[2] client is_active_req TRUE, promote! session a1a48d63-a10b-4d7b-91db-760c654e736b","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.153955142Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
68952023-12-10T18:52:37.021Z{"msg":"[1] ec5eeed3-0492-436e-8824-9bfc0df6805d (a1a48d63-a10b-4d7b-91db-760c654e736b) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.153993334Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
68962023-12-10T18:52:37.021Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.154020519Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
68972023-12-10T18:52:37.021Z{"msg":"[1] client is_active_req TRUE, promote! session a1a48d63-a10b-4d7b-91db-760c654e736b","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.154097393Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
68982023-12-10T18:52:37.021Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.154233037Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
68992023-12-10T18:52:37.022Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.154320631Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69002023-12-10T18:52:37.022Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.154412728Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69012023-12-10T18:52:37.022Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.154516344Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69022023-12-10T18:52:37.023Z{"msg":"ec5eeed3-0492-436e-8824-9bfc0df6805d WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.15459009Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69032023-12-10T18:52:37.023Z{"msg":"ec5eeed3-0492-436e-8824-9bfc0df6805d WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.154653006Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69042023-12-10T18:52:37.023Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.154773872Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69052023-12-10T18:52:37.023Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.154809053Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69062023-12-10T18:52:37.023Z{"msg":"ec5eeed3-0492-436e-8824-9bfc0df6805d WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.154835521Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69072023-12-10T18:52:37.023Z{"msg":"[2] ec5eeed3-0492-436e-8824-9bfc0df6805d (a1a48d63-a10b-4d7b-91db-760c654e736b) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.155247666Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69082023-12-10T18:52:37.023Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.155325125Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69092023-12-10T18:52:37.024Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:36.155435972Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69102023-12-10T18:52:37.025Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.155492501Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69112023-12-10T18:52:37.025Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.15551716Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69122023-12-10T18:52:37.025Z{"msg":"[0] ec5eeed3-0492-436e-8824-9bfc0df6805d (a1a48d63-a10b-4d7b-91db-760c654e736b) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.155546643Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69132023-12-10T18:52:37.025Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.155666268Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69142023-12-10T18:52:37.025Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:36.155730566Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69152023-12-10T18:52:37.025Z{"msg":"ec5eeed3-0492-436e-8824-9bfc0df6805d WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.155771582Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69162023-12-10T18:52:37.025Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.155809788Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69172023-12-10T18:52:37.026Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.155868368Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69182023-12-10T18:52:37.026Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.155907843Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69192023-12-10T18:52:37.026Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.15596242Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69202023-12-10T18:52:37.027Z{"msg":"[1] ec5eeed3-0492-436e-8824-9bfc0df6805d (a1a48d63-a10b-4d7b-91db-760c654e736b) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.15600617Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69212023-12-10T18:52:37.027Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.156044697Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69222023-12-10T18:52:37.027Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:36.15610917Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69232023-12-10T18:52:37.027Z{"msg":"ec5eeed3-0492-436e-8824-9bfc0df6805d WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.156150243Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69242023-12-10T18:52:37.027Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.156183842Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69252023-12-10T18:52:37.027ZThe guest has finished waiting for activation with:56
69262023-12-10T18:52:37.027Z{"msg":"Wait for a query_work_queue command to finish before sending IO
69272023-12-10T18:52:37.027Z[0]R flush_numbers[0..12]: [82, 87, 82, 88, 84, 82, 91, 91, 90, 81, 88, 86]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.156244463Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69282023-12-10T18:52:37.027Z{"msg":"[0]R generation[0..12]: [49, 53, 49, 54, 51, 49, 54, 54, 54, 48, 54, 52]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.156285214Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69292023-12-10T18:52:37.028Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.1563423Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69302023-12-10T18:52:37.028Z{"msg":"[1]R flush_numbers[0..12]: [82, 87, 82, 88, 84, 82, 91, 91, 90, 81, 88, 86]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.156383561Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69312023-12-10T18:52:37.029Z{"msg":"[1]R generation[0..12]: [49, 53, 49, 54, 51, 49, 54, 54, 54, 48, 54, 52]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.156420525Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69322023-12-10T18:52:37.029Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.156491268Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69332023-12-10T18:52:37.029Z{"msg":"[2]R flush_numbers[0..12]: [82, 87, 82, 88, 84, 82, 91, 91, 90, 81, 88, 86]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.156531382Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69342023-12-10T18:52:37.029Z{"msg":"[2]R generation[0..12]: [49, 53, 49, 54, 51, 49, 54, 54, 54, 48, 54, 52]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.156592739Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69352023-12-10T18:52:37.029Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.156634132Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69362023-12-10T18:52:37.029Z{"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.156688739Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69372023-12-10T18:52:37.029Z{"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.15672739Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69382023-12-10T18:52:37.030Z{"msg":"Next flush: 93","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.156781137Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69392023-12-10T18:52:37.030Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.156819216Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69402023-12-10T18:52:37.030Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.156873963Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69412023-12-10T18:52:37.030Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.156911602Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69422023-12-10T18:52:37.030Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
69432023-12-10T18:52:37.031Z,"time":"2023-12-10T18:52:36.156972291Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69442023-12-10T18:52:37.031Z{"msg":"ec5eeed3-0492-436e-8824-9bfc0df6805d is now active with session: a1a48d63-a10b-4d7b-91db-760c654e736b","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.15702257Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69452023-12-10T18:52:37.031Z{"msg":"ec5eeed3-0492-436e-8824-9bfc0df6805d Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.157083474Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69462023-12-10T18:52:37.031Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.157123765Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69472023-12-10T18:52:37.031Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.157176012Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69482023-12-10T18:52:37.031Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.157214843Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69492023-12-10T18:52:37.031Z{"msg":"ec5eeed3-0492-436e-8824-9bfc0df6805d Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.157269451Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69502023-12-10T18:52:37.032Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.157307886Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69512023-12-10T18:52:37.032Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.157365831Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69522023-12-10T18:52:37.033Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.157405171Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69532023-12-10T18:52:37.033Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30Loading write count information from file "/tmp/test_repair_verify.data"
69542023-12-10T18:52:37.033Z,"time":"2023-12-10T18:52:36.157458231Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69552023-12-10T18:52:37.034ZRun Repair workload
69562023-12-10T18:52:37.034Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.157498214Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69572023-12-10T18:52:37.034Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.15755306Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69582023-12-10T18:52:37.034Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.157591062Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69592023-12-10T18:52:37.034Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.157627873Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69602023-12-10T18:52:37.034Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.157680576Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69612023-12-10T18:52:37.034Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:36.15771842Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1386,"session_id":"a1a48d63-a10b-4d7b-91db-760c654e736b"}
69622023-12-10T18:52:37.035Z01/30 Write block 7 len 32768 data: 4 4 4 4 4 4 4 4
69632023-12-10T18:52:37.035Z02/30 Write block 207 len 20480 data: 5 4 3 3 4
69642023-12-10T18:52:37.035Z03/30 Read block 70 len 32768
69652023-12-10T18:52:37.035Z04/30 Read block 569 len 32768
69662023-12-10T18:52:37.035Z05/30 Write block 12 len 36864 data: 5 5 5 5 5 7 5 5 3
69672023-12-10T18:52:37.036Z06/30 Write block 558 len 4096 data: 2
69682023-12-10T18:52:37.045Z07/30 Read block 239 len 36864
69692023-12-10T18:52:37.045Z08/30 Read block 299 len 8192
69702023-12-10T18:52:37.045Z09/30 Write block 482 len 40960 data: 2 3 3 3 5 5 7 7 7 8
69712023-12-10T18:52:37.045Z10/30 Write block 345 len 20480 data: 6 5 7 5 6
69722023-12-10T18:52:37.045Z11/30 Read block 380 len 36864
69732023-12-10T18:52:37.045Z12/30 Write block 226 len 28672 data: 6 5 5 5 5 5 5
69742023-12-10T18:52:37.046Z13/30 Read block 240 len 8192
69752023-12-10T18:52:37.046Z14/30 Flush
69762023-12-10T18:52:37.046Z15/30 Write block 500 len 12288 data: 5 5 5
69772023-12-10T18:52:37.046Z16/30 Read block 177 len 24576
69782023-12-10T18:52:37.046Z17/30 Read block 574 len 4096
69792023-12-10T18:52:37.047Z18/30 Write block 152 len 12288 data: 3 2 4
69802023-12-10T18:52:37.047Z19/30 Write block 527 len 36864 data: 5 5 5 6 7 7 6 7 8
69812023-12-10T18:52:37.047Z20/30 Write block 146 len 36864 data: 5 5 6 6 5 4 4 3 5
69822023-12-10T18:52:37.047Z21/30 Read block 589 len 32768
69832023-12-10T18:52:37.048Z22/30 Read block 227 len 24576
69842023-12-10T18:52:37.048Z23/30 Read block 335 len 20480
69852023-12-10T18:52:37.048Z24/30 Write block 119 len 8192 data: 8 9
69862023-12-10T18:52:37.048Z25/30 Read block 144 len 16384
69872023-12-10T18:52:37.048Z26/30 Write block 85 len 40960 data: 9 10 8 7 6 5 4 3 4 4
69882023-12-10T18:52:37.048Z27/30 Read block 310 len 4096
69892023-12-10T18:52:37.048Z28/30 Write block 487 len 12288 data: 6 8 8
69902023-12-10T18:52:37.049Z29/30 Read block 548 len 12288
69912023-12-10T18:52:37.049Z30/30 Read block 91 len 24576
69922023-12-10T18:52:37.049Z----------------------------------------------------------------
69932023-12-10T18:52:37.050Z Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30
69942023-12-10T18:52:37.050ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
69952023-12-10T18:52:37.050Z 1 Acked 1000 Write 8 Done Done Done false
69962023-12-10T18:52:37.050Z 2 Acked 1001 Write 5 Done Done Done false
69972023-12-10T18:52:37.051Z 3 Acked 1002 Read 8 Done Done Done false
69982023-12-10T18:52:37.051Z 4 Acked 1003 Read 8 Done Done Done false
69992023-12-10T18:52:37.051Z 5 Acked 1004 Write 9 Done Sent Done false
70002023-12-10T18:52:37.051Z 6 Acked 1005 Write 1 Done Sent Done false
70012023-12-10T18:52:37.051Z 7 Acked 1006 Read 9 Done Sent Done false
70022023-12-10T18:52:37.051Z 8 Acked 1007 Read 2 Done Sent Done false
70032023-12-10T18:52:37.051Z 9 Acked 1008 Write 10 Done Sent Done false
70042023-12-10T18:52:37.051Z 10 Acked 1009 Write 5 Done Sent Done false
70052023-12-10T18:52:37.051Z 11 Acked 1010 Read 9 Done Sent Done false
70062023-12-10T18:52:37.051Z 12 Acked 1011 Write 7 Done Sent Done false
70072023-12-10T18:52:37.051Z 13 Acked 1012 Read 2 Done Sent Done false
70082023-12-10T18:52:37.051Z 14 Acked 1013 Flush 0 Done Sent Done false
70092023-12-10T18:52:37.051Z 15 Acked 1014 Write 3 Done Sent Done false
70102023-12-10T18:52:37.051Z 16 Acked 1015 Read 6 Done Sent Done false
70112023-12-10T18:52:37.051Z 17 Acked 1016 Read 1 Done Sent Done false
70122023-12-10T18:52:37.051Z 18 Acked 1017 Write 3 Done Sent Done false
70132023-12-10T18:52:37.051Z 19 Acked 1018 Write 9 Done Sent Done false
70142023-12-10T18:52:37.051Z 20 Acked 1019 Write 9 Done Sent Done false
70152023-12-10T18:52:37.052Z 21 Acked 1020 Read 8 Done Sent Done false
70162023-12-10T18:52:37.052Z 22 Acked 1021 Read 6 Done Sent Done false
70172023-12-10T18:52:37.052Z 23 Acked 1022 Read 5 Done Sent Done false
70182023-12-10T18:52:37.052Z 24 Acked 1023 Write 2 Done Sent Done false
70192023-12-10T18:52:37.052Z 25 Acked 1024 Read 4 Done Sent Done false
70202023-12-10T18:52:37.052Z 26 Acked 1025 Write 10 Done Sent Done false
70212023-12-10T18:52:37.052Z 27 Acked 1026 Read 1 Done Sent Done false
70222023-12-10T18:52:37.052Z 28 Acked 1027 Write 3 Done Sent Done false
70232023-12-10T18:52:37.052Z 29 Acked 1028 Read 3 Done Sent Done false
70242023-12-10T18:52:37.052Z 30 Acked 1029 Read 6 Done Sent Sent false
70252023-12-10T18:52:37.052Z STATES DS:0 DS:1 DS:2 TOTAL
70262023-12-10T18:52:37.052Z New 0 0 0 0
70272023-12-10T18:52:37.052Z Sent 0 26 1 27
70282023-12-10T18:52:37.052Z Done 30 4 29 63
70292023-12-10T18:52:37.052Z Skipped 0 0 0 0
70302023-12-10T18:52:37.052Z Error 0 0 0 0
70312023-12-10T18:52:37.053ZLast Flush: 1013 0 1013
70322023-12-10T18:52:37.053ZDownstairs last five completed:
70332023-12-10T18:52:37.053ZUpstairs last five completed: 30 29 28 27 26 25
70342023-12-10T18:52:37.053ZWrote out file "/tmp/test_repair_verify.data"
70352023-12-10T18:52:37.053Z
70362023-12-10T18:52:37.053ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
70372023-12-10T18:52:37.053ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70382023-12-10T18:52:37.053Z 0 000-019 56 49 56 93 82 93 F T F <---
70392023-12-10T18:52:37.053Z 1 020-039 56 53 56 93 87 93 F F F <---
70402023-12-10T18:52:37.054Z 2 040-059 49 49 49 82 82 82 F F F
70412023-12-10T18:52:37.054Z 3 060-079 54 54 54 88 88 88 F F F
70422023-12-10T18:52:37.054Z 4 080-099 51 51 51 84 84 84 T F T <---
70432023-12-10T18:52:37.055Z 5 100-119 49 49 49 82 82 82 T F T <---
70442023-12-10T18:52:37.055Z 6 120-139 54 54 54 91 91 91 T F T <---
70452023-12-10T18:52:37.055Z 7 140-159 54 54 54 91 91 91 T F T <---
70462023-12-10T18:52:37.055Z 8 160-179 54 54 54 90 90 90 F F F
70472023-12-10T18:52:37.055Z 9 180-199 48 48 48 81 81 81 F F F
70482023-12-10T18:52:37.055Z 10 200-219 56 54 56 93 88 93 F T F <---
70492023-12-10T18:52:37.055Z 11 220-239 56 52 56 93 86 93 F F F <---
70502023-12-10T18:52:37.055Z 12 240-259 51 51 51 84 84 84 F F F
70512023-12-10T18:52:37.055Z 13 260-279 55 55 55 92 92 92 F F F
70522023-12-10T18:52:37.055Z 14 280-299 52 52 52 85 85 85 F F F
70532023-12-10T18:52:37.055Z 15 300-319 54 54 54 88 88 88 F F F
70542023-12-10T18:52:37.055Z 16 320-339 54 54 54 88 88 88 F F F
70552023-12-10T18:52:37.055Z 17 340-359 56 54 56 93 90 93 F F F <---
70562023-12-10T18:52:37.055Z 18 360-379 54 54 54 88 88 88 F F F
70572023-12-10T18:52:37.055Z 19 380-399 53 53 53 87 87 87 F F F
70582023-12-10T18:52:37.055Z 20 400-419 49 49 49 82 82 82 F F F
70592023-12-10T18:52:37.055Z 21 420-439 49 49 49 82 82 82 F F F
70602023-12-10T18:52:37.055Z 22 440-459 46 46 46 77 77 77 F F F
70612023-12-10T18:52:37.056Z 23 460-479 54 54 54 91 91 91 F F F
70622023-12-10T18:52:37.056Z 24 480-499 56 52 56 93 85 93 T F T <---
70632023-12-10T18:52:37.056Z 25 500-519 52 52 52 86 86 86 T F T <---
70642023-12-10T18:52:37.056Z 26 520-539 52 52 52 86 86 86 T F T <---
70652023-12-10T18:52:37.056Z 27 540-559 56 42 56 93 69 93 F F F <---
70662023-12-10T18:52:37.056Z 28 560-579 50 50 50 83 83 83 F F F
70672023-12-10T18:52:37.056Z 29 580-599 50 50 50 83 83 83 F F F
70682023-12-10T18:52:37.068ZMax gen: 56, Max flush: 93
70692023-12-10T18:52:37.068ZError: Difference in extent metadata found!
70702023-12-10T18:52:37.068ZA Difference in extent metadata is expected here
70712023-12-10T18:52:37.068ZOn loop 28
70722023-12-10T18:52:37.068Z
70732023-12-10T18:52:37.068ZVerifying data now
70742023-12-10T18:52:41.539ZNo change after verify
70752023-12-10T18:52:41.540ZLoop: 28 Downstairs dump after verify (and repair):
70762023-12-10T18:52:41.586ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70772023-12-10T18:52:41.586Z 0 000-019 56 56 56 93 93 93 F F F
70782023-12-10T18:52:41.591Z 1 020-039 56 56 56 93 93 93 F F F
70792023-12-10T18:52:41.591Z 2 040-059 49 49 49 82 82 82 F F F
70802023-12-10T18:52:41.591Z 3 060-079 54 54 54 88 88 88 F F F
70812023-12-10T18:52:41.592Z 4 080-099 57 57 57 94 94 94 F F F
70822023-12-10T18:52:41.592Z 5 100-119 57 57 57 94 94 94 F F F
70832023-12-10T18:52:41.592Z 6 120-139 57 57 57 94 94 94 F F F
70842023-12-10T18:52:41.592Z 7 140-159 57 57 57 94 94 94 F F F
70852023-12-10T18:52:41.592Z 8 160-179 54 54 54 90 90 90 F F F
70862023-12-10T18:52:41.592Z 9 180-199 48 48 48 81 81 81 F F F
70872023-12-10T18:52:41.592Z 10 200-219 56 56 56 93 93 93 F F F
70882023-12-10T18:52:41.592Z 11 220-239 56 56 56 93 93 93 F F F
70892023-12-10T18:52:41.592Z 12 240-259 51 51 51 84 84 84 F F F
70902023-12-10T18:52:41.592Z 13 260-279 55 55 55 92 92 92 F F F
70912023-12-10T18:52:41.592Z 14 280-299 52 52 52 85 85 85 F F F
70922023-12-10T18:52:41.592Z 15 300-319 54 54 54 88 88 88 F F F
70932023-12-10T18:52:41.592Z 16 320-339 54 54 54 88 88 88 F F F
70942023-12-10T18:52:41.592Z 17 340-359 56 56 56 93 93 93 F F F
70952023-12-10T18:52:41.592Z 18 360-379 54 54 54 88 88 88 F F F
70962023-12-10T18:52:41.592Z 19 380-399 53 53 53 87 87 87 F F F
70972023-12-10T18:52:41.592Z 20 400-419 49 49 49 82 82 82 F F F
70982023-12-10T18:52:41.592Z 21 420-439 49 49 49 82 82 82 F F F
70992023-12-10T18:52:41.593Z 22 440-459 46 46 46 77 77 77 F F F
71002023-12-10T18:52:41.593Z 23 460-479 54 54 54 91 91 91 F F F
71012023-12-10T18:52:41.593Z 24 480-499 57 57 57 94 94 94 F F F
71022023-12-10T18:52:41.593Z 25 500-519 57 57 57 94 94 94 F F F
71032023-12-10T18:52:41.593Z 26 520-539 57 57 57 94 94 94 F F F
71042023-12-10T18:52:41.593Z 27 540-559 56 56 56 93 93 93 F F F
71052023-12-10T18:52:41.593Z 28 560-579 50 50 50 83 83 83 F F F
71062023-12-10T18:52:41.594Z 29 580-599 50 50 50 83 83 83 F F F
71072023-12-10T18:52:41.594ZMax gen: 57, Max flush: 94
71082023-12-10T18:52:41.594Z
71092023-12-10T18:52:41.594ZBegin loop 29
71102023-12-10T18:52:41.594ZDownstairs to restart: 1
71112023-12-10T18:52:41.597Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.598263727Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394}
71122023-12-10T18:52:41.597Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.598780296Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394}
71132023-12-10T18:52:41.598Z{"msg":"Upstairs <-> Downstairs Message Version: 5"Crucible runtime is spawned
71142023-12-10T18:52:41.598Z,"v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.599019167Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394}
71152023-12-10T18:52:41.598Z{"msg":"The guest has requested activation with gen:58
71162023-12-10T18:52:41.598ZCrucible d52b5b0a-2b39-4ece-a0dd-bb23b33f91cb has session id: a7326200-6bcb-42f7-bba6-07df544e58c9","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.599163721Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71172023-12-10T18:52:41.598Z{"msg":"Upstairs opts: Upstairs UUID: d52b5b0a-2b39-4ece-a0dd-bb23b33f91cb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.599251436Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71182023-12-10T18:52:41.598Z{"msg":"Crucible stats registered with UUID: d52b5b0a-2b39-4ece-a0dd-bb23b33f91cb","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.599290762Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71192023-12-10T18:52:41.598Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.599361927Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9","task":"up_listen"}
71202023-12-10T18:52:41.598Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.59940468Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71212023-12-10T18:52:41.598Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.599462925Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71222023-12-10T18:52:41.598Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.599561582Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"looper":"1","session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71232023-12-10T18:52:41.598Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.599622322Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"looper":"0","session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71242023-12-10T18:52:41.604Z{"msg":"[1] d52b5b0a-2b39-4ece-a0dd-bb23b33f91cb looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.599665122Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"looper":"1","session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71252023-12-10T18:52:41.604Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.599767263Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"looper":"2","session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71262023-12-10T18:52:41.604Z{"msg":"[0] d52b5b0a-2b39-4ece-a0dd-bb23b33f91cb looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.599794885Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"looper":"0","session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71272023-12-10T18:52:41.604Z{"msg":"[2] d52b5b0a-2b39-4ece-a0dd-bb23b33f91cb looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.599853066Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"looper":"2","session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71282023-12-10T18:52:41.605Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.600164295Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71292023-12-10T18:52:41.605Z{"msg":"d52b5b0a-2b39-4ece-a0dd-bb23b33f91cb active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.60024124Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71302023-12-10T18:52:41.605Z{"msg":"Set desired generation to :58","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.600278719Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71312023-12-10T18:52:41.605Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.600579936Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71322023-12-10T18:52:41.605Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.600742779Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71332023-12-10T18:52:41.605Z{"msg":"[2] d52b5b0a-2b39-4ece-a0dd-bb23b33f91cb (a7326200-6bcb-42f7-bba6-07df544e58c9) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.601183458Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71342023-12-10T18:52:41.605Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.601248099Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71352023-12-10T18:52:41.605Z{"msg":"[2] client is_active_req TRUE, promote! session a7326200-6bcb-42f7-bba6-07df544e58c9","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.601274724Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71362023-12-10T18:52:41.605Z{"msg":"[0] d52b5b0a-2b39-4ece-a0dd-bb23b33f91cb (a7326200-6bcb-42f7-bba6-07df544e58c9) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.601344292Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71372023-12-10T18:52:41.605Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.601369007Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71382023-12-10T18:52:41.605Z{"msg":"[0] client is_active_req TRUE, promote! session a7326200-6bcb-42f7-bba6-07df544e58c9","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.601403739Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71392023-12-10T18:52:41.605Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.601898804Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71402023-12-10T18:52:41.605Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.602062794Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71412023-12-10T18:52:41.605Z{"msg":"d52b5b0a-2b39-4ece-a0dd-bb23b33f91cb WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.602116408Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71422023-12-10T18:52:41.605Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.602200285Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71432023-12-10T18:52:41.606Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.602240689Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71442023-12-10T18:52:41.606Z{"msg":"d52b5b0a-2b39-4ece-a0dd-bb23b33f91cb WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.602285897Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71452023-12-10T18:52:41.606Z{"msg":"[2] d52b5b0a-2b39-4ece-a0dd-bb23b33f91cb (a7326200-6bcb-42f7-bba6-07df544e58c9) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.602954695Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71462023-12-10T18:52:41.606Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.603028115Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71472023-12-10T18:52:41.606Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:41.603053013Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71482023-12-10T18:52:41.606Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.603094265Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71492023-12-10T18:52:41.606Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.603127718Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71502023-12-10T18:52:41.606Z{"msg":"d52b5b0a-2b39-4ece-a0dd-bb23b33f91cb WaitActive New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.603170787Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71512023-12-10T18:52:41.606Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.603193704Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71522023-12-10T18:52:41.606Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.603213927Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71532023-12-10T18:52:41.606Z{"msg":"[0] d52b5b0a-2b39-4ece-a0dd-bb23b33f91cb (a7326200-6bcb-42f7-bba6-07df544e58c9) WaitActive New WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.603396065Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71542023-12-10T18:52:41.606Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.603506838Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71552023-12-10T18:52:41.606Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:41.603551735Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71562023-12-10T18:52:41.606Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.603574659Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71572023-12-10T18:52:41.606Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.603594569Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71582023-12-10T18:52:41.607Z{"msg":"d52b5b0a-2b39-4ece-a0dd-bb23b33f91cb WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.603649462Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71592023-12-10T18:52:41.607Z{"msg":"[1] d52b5b0a-2b39-4ece-a0dd-bb23b33f91cb (a7326200-6bcb-42f7-bba6-07df544e58c9) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.603674599Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71602023-12-10T18:52:41.607Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.603695072Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71612023-12-10T18:52:41.607Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.603725079Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71622023-12-10T18:52:41.607Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.603762358Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71632023-12-10T18:52:41.607Z{"msg":"[1] client is_active_req TRUE, promote! session a7326200-6bcb-42f7-bba6-07df544e58c9","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.603835693Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71642023-12-10T18:52:41.612Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.604427496Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71652023-12-10T18:52:41.612Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.604502001Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71662023-12-10T18:52:41.612Z{"msg":"d52b5b0a-2b39-4ece-a0dd-bb23b33f91cb WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.604560993Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71672023-12-10T18:52:41.612Z{"msg":"[1] d52b5b0a-2b39-4ece-a0dd-bb23b33f91cb (a7326200-6bcb-42f7-bba6-07df544e58c9) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.605039431Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71682023-12-10T18:52:41.614Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.605105703Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71692023-12-10T18:52:41.614Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:41.605129404Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71702023-12-10T18:52:41.614Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.605149656Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71712023-12-10T18:52:41.614Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.605170077Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71722023-12-10T18:52:41.614Z{"msg":"d52b5b0a-2b39-4ece-a0dd-bb23b33f91cb WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.60519074Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71732023-12-10T18:52:41.614Z{"msg":"[0]R flush_numbers[0..12]: [93, 93, 82, 88, 94, 94, 94, 94, 90, 81, 93, 93]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.605222763Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71742023-12-10T18:52:41.614Z{"msg":"[0]R generation[0..12]: [56, 56, 49, 54, 57, 57, 57, 57, 54, 48, 56, 56]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.605253761Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71752023-12-10T18:52:41.614Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.605297672Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71762023-12-10T18:52:41.614Z{"msg":"[1]R flush_numbers[0..12]: [93, 93, 82, 88, 94, 94, 94, 94, 90, 81, 93, 93]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.605321204Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71772023-12-10T18:52:41.615Z{"msg":"[1]R generation[0..12]: [56, 56, 49, 54, 57, 57, 57, 57, 54, 48, 56, 56]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.605342665Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71782023-12-10T18:52:41.615Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.605385019Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71792023-12-10T18:52:41.615Z{"msg":"[2]R flush_numbers[0..12]: [93, 93, 82, 88, 94, 94, 94, 94, 90, 81, 93, 93]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.605408617Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71802023-12-10T18:52:41.615Z{"msg":"[2]R generation[0..12]: [56, 56, 49, 54, 57, 57, 57, 57, 54, 48, 56, 56]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.605451914Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71812023-12-10T18:52:41.615Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.605475966Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71822023-12-10T18:52:41.615Z{"msg":"Max found gen is 58","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:58
71832023-12-10T18:52:41.615ZWait for a query_work_queue command to finish before sending IO
71842023-12-10T18:52:41.615Z,"time":"2023-12-10T18:52:41.605534687Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71852023-12-10T18:52:41.615Z{"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.605708566Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71862023-12-10T18:52:41.615Z{"msg":"Next flush: 95","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.605798655Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71872023-12-10T18:52:41.615Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.605834909Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71882023-12-10T18:52:41.615Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.605863115Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71892023-12-10T18:52:41.615Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.605908504Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71902023-12-10T18:52:41.617Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.605931502Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71912023-12-10T18:52:41.617Z{"msg":"d52b5b0a-2b39-4ece-a0dd-bb23b33f91cb is now active with session: a7326200-6bcb-42f7-bba6-07df544e58c9","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.605954969Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71922023-12-10T18:52:41.617Z{"msg":"d52b5b0a-2b39-4ece-a0dd-bb23b33f91cb Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.605996037Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71932023-12-10T18:52:41.617Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.606018889Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71942023-12-10T18:52:41.617Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.606059976Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71952023-12-10T18:52:41.617Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.606081719Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71962023-12-10T18:52:41.617Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.606105084Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71972023-12-10T18:52:41.617Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.606162511Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71982023-12-10T18:52:41.617Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.606208546Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
71992023-12-10T18:52:41.617Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.60627619Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
72002023-12-10T18:52:41.617Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.606345345Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
72012023-12-10T18:52:41.618Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.606413036Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
72022023-12-10T18:52:41.618ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
72032023-12-10T18:52:41.618Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.606687059Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
72042023-12-10T18:52:41.618Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:41.60674329Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1394,"session_id":"a7326200-6bcb-42f7-bba6-07df544e58c9"}
72052023-12-10T18:52:41.618ZLoading write count information from file "/tmp/test_repair_verify.data"
72062023-12-10T18:52:41.619ZRun Repair workload
72072023-12-10T18:52:41.619Z01/30 Write block 13 len 32768 data: 6 6 6 6 8 6 6 4
72082023-12-10T18:52:41.619Z02/30 Read block 147 len 20480
72092023-12-10T18:52:41.619Z03/30 Write block 102 len 20480 data: 5 5 4 4 5
72102023-12-10T18:52:41.619Z04/30 Read block 373 len 36864
72112023-12-10T18:52:41.622Z05/30 Flush
72122023-12-10T18:52:41.622Z06/30 Write block 465 len 16384 data: 7 6 7 7
72132023-12-10T18:52:41.622Z07/30 Read block 246 len 32768
72142023-12-10T18:52:41.623Z08/30 Read block 507 len 36864
72152023-12-10T18:52:41.626Z09/30 Write block 337 len 32768 data: 3 3 3 2 3 5 6 7
72162023-12-10T18:52:41.626Z10/30 Flush
72172023-12-10T18:52:41.629Z11/30 Write block 36 len 28672 data: 6 6 5 6 5 5 5
72182023-12-10T18:52:41.632Z12/30 Write block 285 len 40960 data: 8 8 7 6 4 3 4 4 5 5
72192023-12-10T18:52:41.632Z13/30 Write block 573 len 8192 data: 3 4
72202023-12-10T18:52:41.632Z14/30 Flush
72212023-12-10T18:52:41.635Z15/30 Write block 124 len 20480 data: 12 10 9 8 8
72222023-12-10T18:52:41.643Z16/30 Write block 144 len 8192 data: 4 4
72232023-12-10T18:52:41.643Z17/30 Write block 498 len 24576 data: 2 3 6 6 6 7
72242023-12-10T18:52:41.643Z18/30 Read block 306 len 28672
72252023-12-10T18:52:41.644Z19/30 Read block 140 len 4096
72262023-12-10T18:52:41.644Z20/30 Write block 321 len 36864 data: 5 7 6 7 6 7 7 7 6
72272023-12-10T18:52:41.644Z21/30 Read block 340 len 28672
72282023-12-10T18:52:41.647Z22/30 Write block 558 len 12288 data: 3 3 3
72292023-12-10T18:52:41.647Z23/30 Read block 485 len 24576
72302023-12-10T18:52:41.651Z24/30 Write block 291 len 8192 data: 5 5
72312023-12-10T18:52:41.651Z25/30 Write block 507 len 16384 data: 5 5 4 4
72322023-12-10T18:52:41.651Z26/30 Write block 437 len 36864 data: 3 4 4 6 7 4 4 3 4
72332023-12-10T18:52:41.655Z27/30 Write block 303 len 36864 data: 7 8 5 5 3 3 4 2 2
72342023-12-10T18:52:41.655Z28/30 Write block 378 len 36864 data: 8 9 9 6 7 7 8 8 7
72352023-12-10T18:52:41.655Z29/30 Read block 144 len 16384
72362023-12-10T18:52:41.658Z30/30 Read block 89 len 36864
72372023-12-10T18:52:41.662Z----------------------------------------------------------------
72382023-12-10T18:52:41.662Z Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:20
72392023-12-10T18:52:41.662ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
72402023-12-10T18:52:41.662Z 11 Acked 1010 Write 7 Done Done Done false
72412023-12-10T18:52:41.662Z 12 Acked 1011 Write 10 Done Done Done false
72422023-12-10T18:52:41.662Z 13 Acked 1012 Write 2 Done Sent Done false
72432023-12-10T18:52:41.663Z 14 Acked 1013 Flush 0 Done Sent Done false
72442023-12-10T18:52:41.663Z 15 Acked 1014 Write 5 Done Sent Done false
72452023-12-10T18:52:41.663Z 16 Acked 1015 Write 2 Done Sent Done false
72462023-12-10T18:52:41.663Z 17 Acked 1016 Write 6 Done Sent Done false
72472023-12-10T18:52:41.663Z 18 Acked 1017 Read 7 Done Sent Done false
72482023-12-10T18:52:41.663Z 19 Acked 1018 Read 1 Done Sent Done false
72492023-12-10T18:52:41.663Z 20 Acked 1019 Write 9 Done Sent Done false
72502023-12-10T18:52:41.663Z 21 Acked 1020 Read 7 Done Sent Done false
72512023-12-10T18:52:41.663Z 22 Acked 1021 Write 3 Done Sent Done false
72522023-12-10T18:52:41.663Z 23 Acked 1022 Read 6 Done Sent Done false
72532023-12-10T18:52:41.664Z 24 Acked 1023 Write 2 Done Sent Done false
72542023-12-10T18:52:41.664Z 25 Acked 1024 Write 4 Done Sent Done false
72552023-12-10T18:52:41.664Z 26 Acked 1025 Write 9 Done Sent Done false
72562023-12-10T18:52:41.664Z 27 Acked 1026 Write 9 Done Sent Done false
72572023-12-10T18:52:41.664Z 28 Acked 1027 Write 9 Done Sent Done false
72582023-12-10T18:52:41.664Z 29 Acked 1028 Read 4 Done Sent Done false
72592023-12-10T18:52:41.664Z 30 Acked 1029 Read 9 Sent Sent Done false
72602023-12-10T18:52:41.664Z STATES DS:0 DS:1 DS:2 TOTAL
72612023-12-10T18:52:41.664Z New 0 0 0 0
72622023-12-10T18:52:41.664Z Sent 1 18 0 19
72632023-12-10T18:52:41.665Z Done 19 2 20 41
72642023-12-10T18:52:41.665Z Skipped 0 0 0 0
72652023-12-10T18:52:41.665Z Error 0 0 0 0
72662023-12-10T18:52:41.665ZLast Flush: 1013 1009 1013
72672023-12-10T18:52:41.665ZDownstairs last five completed: 1009 1008 1007 1006 1005 1004
72682023-12-10T18:52:41.665ZUpstairs last five completed: 30 29 28 27 26 25
72692023-12-10T18:52:41.665ZWrote out file "/tmp/test_repair_verify.data"
72702023-12-10T18:52:41.671Z
72712023-12-10T18:52:41.675ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
72722023-12-10T18:52:41.705ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72732023-12-10T18:52:41.705Z 0 000-019 58 58 58 95 95 95 F F F
72742023-12-10T18:52:41.709Z 1 020-039 58 58 58 97 95 97 F T F <---
72752023-12-10T18:52:41.709Z 2 040-059 58 49 58 97 82 97 F T F <---
72762023-12-10T18:52:41.712Z 3 060-079 54 54 54 88 88 88 F F F
72772023-12-10T18:52:41.712Z 4 080-099 57 57 57 94 94 94 F F F
72782023-12-10T18:52:41.713Z 5 100-119 58 58 58 95 95 95 F F F
72792023-12-10T18:52:41.713Z 6 120-139 57 57 57 94 94 94 T F T <---
72802023-12-10T18:52:41.713Z 7 140-159 57 57 57 94 94 94 T F T <---
72812023-12-10T18:52:41.713Z 8 160-179 54 54 54 90 90 90 F F F
72822023-12-10T18:52:41.713Z 9 180-199 48 48 48 81 81 81 F F F
72832023-12-10T18:52:41.713Z 10 200-219 56 56 56 93 93 93 F F F
72842023-12-10T18:52:41.713Z 11 220-239 56 56 56 93 93 93 F F F
72852023-12-10T18:52:41.713Z 12 240-259 51 51 51 84 84 84 F F F
72862023-12-10T18:52:41.713Z 13 260-279 55 55 55 92 92 92 F F F
72872023-12-10T18:52:41.713Z 14 280-299 58 52 58 97 85 97 T T T <---
72882023-12-10T18:52:41.713Z 15 300-319 54 54 54 88 88 88 T F T <---
72892023-12-10T18:52:41.714Z 16 320-339 58 58 58 96 96 96 T F T <---
72902023-12-10T18:52:41.714Z 17 340-359 58 58 58 96 96 96 F F F
72912023-12-10T18:52:41.714Z 18 360-379 54 54 54 88 88 88 T F T <---
72922023-12-10T18:52:41.714Z 19 380-399 53 53 53 87 87 87 T F T <---
72932023-12-10T18:52:41.714Z 20 400-419 49 49 49 82 82 82 F F F
72942023-12-10T18:52:41.714Z 21 420-439 49 49 49 82 82 82 T F T <---
72952023-12-10T18:52:41.714Z 22 440-459 46 46 46 77 77 77 T F T <---
72962023-12-10T18:52:41.714Z 23 460-479 58 58 58 96 96 96 F F F
72972023-12-10T18:52:41.714Z 24 480-499 57 57 57 94 94 94 T F T <---
72982023-12-10T18:52:41.714Z 25 500-519 57 57 57 94 94 94 T F T <---
72992023-12-10T18:52:41.714Z 26 520-539 57 57 57 94 94 94 F F F
73002023-12-10T18:52:41.714Z 27 540-559 56 56 56 93 93 93 T F T <---
73012023-12-10T18:52:41.715Z 28 560-579 58 50 58 97 83 97 T F T <---
73022023-12-10T18:52:41.715Z 29 580-599 50 50 50 83 83 83 F F F
73032023-12-10T18:52:41.715ZMax gen: 58, Max flush: 97
73042023-12-10T18:52:41.715ZError: Difference in extent metadata found!
73052023-12-10T18:52:41.719ZA Difference in extent metadata is expected here
73062023-12-10T18:52:41.719ZOn loop 29
73072023-12-10T18:52:41.719Z
73082023-12-10T18:52:41.719ZVerifying data now
73092023-12-10T18:52:48.174ZNo change after verify
73102023-12-10T18:52:48.175ZLoop: 29 Downstairs dump after verify (and repair):
73112023-12-10T18:52:48.221ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
73122023-12-10T18:52:48.222Z 0 000-019 58 58 58 95 95 95 F F F
73132023-12-10T18:52:48.243Z 1 020-039 58 58 58 97 97 97 F F F
73142023-12-10T18:52:48.244Z 2 040-059 58 58 58 97 97 97 F F F
73152023-12-10T18:52:48.244Z 3 060-079 54 54 54 88 88 88 F F F
73162023-12-10T18:52:48.244Z 4 080-099 57 57 57 94 94 94 F F F
73172023-12-10T18:52:48.244Z 5 100-119 58 58 58 95 95 95 F F F
73182023-12-10T18:52:48.244Z 6 120-139 59 59 59 98 98 98 F F F
73192023-12-10T18:52:48.245Z 7 140-159 59 59 59 98 98 98 F F F
73202023-12-10T18:52:48.245Z 8 160-179 54 54 54 90 90 90 F F F
73212023-12-10T18:52:48.245Z 9 180-199 48 48 48 81 81 81 F F F
73222023-12-10T18:52:48.245Z 10 200-219 56 56 56 93 93 93 F F F
73232023-12-10T18:52:48.245Z 11 220-239 56 56 56 93 93 93 F F F
73242023-12-10T18:52:48.245Z 12 240-259 51 51 51 84 84 84 F F F
73252023-12-10T18:52:48.245Z 13 260-279 55 55 55 92 92 92 F F F
73262023-12-10T18:52:48.246Z 14 280-299 59 59 59 98 98 98 F F F
73272023-12-10T18:52:48.246Z 15 300-319 59 59 59 98 98 98 F F F
73282023-12-10T18:52:48.246Z 16 320-339 59 59 59 98 98 98 F F F
73292023-12-10T18:52:48.246Z 17 340-359 58 58 58 96 96 96 F F F
73302023-12-10T18:52:48.246Z 18 360-379 59 59 59 98 98 98 F F F
73312023-12-10T18:52:48.246Z 19 380-399 59 59 59 98 98 98 F F F
73322023-12-10T18:52:48.247Z 20 400-419 49 49 49 82 82 82 F F F
73332023-12-10T18:52:48.247Z 21 420-439 59 59 59 98 98 98 F F F
73342023-12-10T18:52:48.247Z 22 440-459 59 59 59 98 98 98 F F F
73352023-12-10T18:52:48.247Z 23 460-479 58 58 58 96 96 96 F F F
73362023-12-10T18:52:48.247Z 24 480-499 59 59 59 98 98 98 F F F
73372023-12-10T18:52:48.247Z 25 500-519 59 59 59 98 98 98 F F F
73382023-12-10T18:52:48.248Z 26 520-539 57 57 57 94 94 94 F F F
73392023-12-10T18:52:48.248Z 27 540-559 59 59 59 98 98 98 F F F
73402023-12-10T18:52:48.248Z 28 560-579 59 59 59 98 98 98 F F F
73412023-12-10T18:52:48.248Z 29 580-599 50 50 50 83 83 83 F F F
73422023-12-10T18:52:48.248ZMax gen: 59, Max flush: 98
73432023-12-10T18:52:48.248Z
73442023-12-10T18:52:48.249ZBegin loop 30
73452023-12-10T18:52:48.249ZDownstairs to restart: 0
73462023-12-10T18:52:48.265Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.236401885Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402}
73472023-12-10T18:52:48.266Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.23687286Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402}
73482023-12-10T18:52:48.266Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.236951015Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402}
73492023-12-10T18:52:48.266Z{"msg":"Crucible runtime is spawned
73502023-12-10T18:52:48.266ZCrucible f3f863d3-ad52-403a-8fe4-33a8c147e42e has session id: a6ff3b69-a92d-4695-a3ad-2448d445baa1","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.236998248Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73512023-12-10T18:52:48.266Z{"msg":"The guest has requested activation with gen:60
73522023-12-10T18:52:48.267ZUpstairs opts: Upstairs UUID: f3f863d3-ad52-403a-8fe4-33a8c147e42e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.237092435Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73532023-12-10T18:52:48.267Z{"msg":"Crucible stats registered with UUID: f3f863d3-ad52-403a-8fe4-33a8c147e42e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.237137147Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73542023-12-10T18:52:48.268Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.237175185Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"looper":"0","session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73552023-12-10T18:52:48.268Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.237257372Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1","task":"up_listen"}
73562023-12-10T18:52:48.268Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.237300241Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73572023-12-10T18:52:48.268Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.23733536Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73582023-12-10T18:52:48.268Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.237559913Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"looper":"1","session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73592023-12-10T18:52:48.269Z{"msg":"[0] f3f863d3-ad52-403a-8fe4-33a8c147e42e looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.23764061Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"looper":"0","session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73602023-12-10T18:52:48.269Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.237734275Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"looper":"2","session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73612023-12-10T18:52:48.269Z{"msg":"[1] f3f863d3-ad52-403a-8fe4-33a8c147e42e looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.237821705Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"looper":"1","session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73622023-12-10T18:52:48.269Z{"msg":"[2] f3f863d3-ad52-403a-8fe4-33a8c147e42e looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.237970104Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"looper":"2","session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73632023-12-10T18:52:48.269Z{"msg":"f3f863d3-ad52-403a-8fe4-33a8c147e42e active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.238236697Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73642023-12-10T18:52:48.270Z{"msg":"Set desired generation to :60","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.238289594Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73652023-12-10T18:52:48.270Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.238386349Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73662023-12-10T18:52:48.270Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.238613101Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73672023-12-10T18:52:48.270Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.238682973Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73682023-12-10T18:52:48.270Z{"msg":"[1] f3f863d3-ad52-403a-8fe4-33a8c147e42e (a6ff3b69-a92d-4695-a3ad-2448d445baa1) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.239093187Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73692023-12-10T18:52:48.270Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.23915674Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73702023-12-10T18:52:48.270Z{"msg":"[1] client is_active_req TRUE, promote! session a6ff3b69-a92d-4695-a3ad-2448d445baa1","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.239257438Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73712023-12-10T18:52:48.271Z{"msg":"[2] f3f863d3-ad52-403a-8fe4-33a8c147e42e (a6ff3b69-a92d-4695-a3ad-2448d445baa1) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.239333898Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73722023-12-10T18:52:48.271Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.239372146Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73732023-12-10T18:52:48.271Z{"msg":"[2] client is_active_req TRUE, promote! session a6ff3b69-a92d-4695-a3ad-2448d445baa1","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.239408425Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73742023-12-10T18:52:48.271Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.240056081Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73752023-12-10T18:52:48.272Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.240123036Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73762023-12-10T18:52:48.272Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.240152084Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73772023-12-10T18:52:48.272Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.240185265Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73782023-12-10T18:52:48.273Z{"msg":"f3f863d3-ad52-403a-8fe4-33a8c147e42e New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.240393833Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73792023-12-10T18:52:48.273Z{"msg":"f3f863d3-ad52-403a-8fe4-33a8c147e42e New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.240475263Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73802023-12-10T18:52:48.273Z{"msg":"[0] f3f863d3-ad52-403a-8fe4-33a8c147e42e (a6ff3b69-a92d-4695-a3ad-2448d445baa1) New WaitActive WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.240522188Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73812023-12-10T18:52:48.273Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.240614237Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73822023-12-10T18:52:48.274Z{"msg":"[0] client is_active_req TRUE, promote! session a6ff3b69-a92d-4695-a3ad-2448d445baa1","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.240641842Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73832023-12-10T18:52:48.274Z{"msg":"[1] f3f863d3-ad52-403a-8fe4-33a8c147e42e (a6ff3b69-a92d-4695-a3ad-2448d445baa1) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.240902107Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73842023-12-10T18:52:48.275Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.240981463Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73852023-12-10T18:52:48.275Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:48.241076963Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73862023-12-10T18:52:48.275Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.241104074Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73872023-12-10T18:52:48.275Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.241124886Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73882023-12-10T18:52:48.276Z{"msg":"f3f863d3-ad52-403a-8fe4-33a8c147e42e WaitActive WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.241159354Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73892023-12-10T18:52:48.276Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.241277972Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73902023-12-10T18:52:48.276Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.241382612Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73912023-12-10T18:52:48.276Z{"msg":"[2] f3f863d3-ad52-403a-8fe4-33a8c147e42e (a6ff3b69-a92d-4695-a3ad-2448d445baa1) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.241506293Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73922023-12-10T18:52:48.278Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.241530777Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73932023-12-10T18:52:48.279Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:48.241550971Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73942023-12-10T18:52:48.279Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.241570457Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73952023-12-10T18:52:48.279Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.241591037Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73962023-12-10T18:52:48.279Z{"msg":"f3f863d3-ad52-403a-8fe4-33a8c147e42e WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.241611372Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73972023-12-10T18:52:48.280Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.241631523Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73982023-12-10T18:52:48.280Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.241671359Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
73992023-12-10T18:52:48.280Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.241694608Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74002023-12-10T18:52:48.281Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.2417394Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74012023-12-10T18:52:48.281Z{"msg":"f3f863d3-ad52-403a-8fe4-33a8c147e42e WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.241762773Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74022023-12-10T18:52:48.282Z{"msg":"[0] f3f863d3-ad52-403a-8fe4-33a8c147e42e (a6ff3b69-a92d-4695-a3ad-2448d445baa1) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.242234887Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74032023-12-10T18:52:48.283Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.242309868Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74042023-12-10T18:52:48.283Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:48.242332921Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74052023-12-10T18:52:48.283Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.242352711Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74062023-12-10T18:52:48.283Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.242373116Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74072023-12-10T18:52:48.284Z{"msg":"f3f863d3-ad52-403a-8fe4-33a8c147e42e WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.242394066Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74082023-12-10T18:52:48.284Z{"msg":"[0]R flush_numbers[0..12]: [95, 97, 97, 88, 94, 95, 98, 98, 90, 81, 93, 93]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.242434878Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74092023-12-10T18:52:48.284Z{"msg":"[0]R generation[0..12]: [58, 58, 58, 54, 57, 58, 59, 59, 54, 48, 56, 56]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.242457975Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74102023-12-10T18:52:48.284Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.242479456Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74112023-12-10T18:52:48.285Z{"msg":"[1]R flush_numbers[0..12]: [95, 97, 97, 88, 94, 95, 98, 98, 90, 81, 93, 93]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.242520557Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74122023-12-10T18:52:48.287Z{"msg":"[1]R generation[0..12]: [58, 58, 58, 54, 57, 58, 59, 59, 54, 48, 56, 56]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.242543786Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74132023-12-10T18:52:48.287Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.242566002Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74142023-12-10T18:52:48.287Z{"msg":"[2]R flush_numbers[0..12]: [95, 97, 97, 88, 94, 95, 98, 98, 90, 81, 93, 93]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.242587365Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74152023-12-10T18:52:48.288Z{"msg":"[2]R generation[0..12]: [58, 58, 58, 54, 57, 58, 59, 59, 54, 48, 56, 56]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.242607937Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1The guest has finished waiting for activation with:60
74162023-12-10T18:52:48.288ZWait for a query_work_queue command to finish before sending IO
74172023-12-10T18:52:48.288Z"}
74182023-12-10T18:52:48.288Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.242694471Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74192023-12-10T18:52:48.288Z{"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.242741611Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74202023-12-10T18:52:48.289Z{"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.242771484Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74212023-12-10T18:52:48.289Z{"msg":"Next flush: 99","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.242807236Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74222023-12-10T18:52:48.289Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.242843252Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74232023-12-10T18:52:48.290Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.242910879Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74242023-12-10T18:52:48.290Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.24293664Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74252023-12-10T18:52:48.290Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.242956857Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74262023-12-10T18:52:48.291Z{"msg":"f3f863d3-ad52-403a-8fe4-33a8c147e42e is now active with session: a6ff3b69-a92d-4695-a3ad-2448d445baa1","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.242979333Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74272023-12-10T18:52:48.291Z{"msg":"f3f863d3-ad52-403a-8fe4-33a8c147e42e Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.243001679Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74282023-12-10T18:52:48.291Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.243021894Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74292023-12-10T18:52:48.291Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.243041313Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74302023-12-10T18:52:48.291Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.243060696Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74312023-12-10T18:52:48.292Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.243080034Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74322023-12-10T18:52:48.292Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.243134099Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74332023-12-10T18:52:48.292Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.243388353Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74342023-12-10T18:52:48.292Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.243445032Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74352023-12-10T18:52:48.293Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.243744803Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74362023-12-10T18:52:48.293Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.243808342Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74372023-12-10T18:52:48.293Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:48.243846562Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74382023-12-10T18:52:48.293Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
74392023-12-10T18:52:48.293Z,"time":"2023-12-10T18:52:48.243898623Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1402,"session_id":"a6ff3b69-a92d-4695-a3ad-2448d445baa1"}
74402023-12-10T18:52:48.293ZLoading write count information from file "/tmp/test_repair_verify.data"
74412023-12-10T18:52:48.294ZRun Repair workload
74422023-12-10T18:52:48.294Z01/30 Flush
74432023-12-10T18:52:48.294Z02/30 Write block 188 len 36864 data: 4 4 4 6 6 6 6 8 6
74442023-12-10T18:52:48.294Z03/30 Read block 552 len 32768
74452023-12-10T18:52:48.294Z04/30 Read block 569 len 20480
74462023-12-10T18:52:48.303Z05/30 Read block 524 len 28672
74472023-12-10T18:52:48.304Z06/30 Write block 574 len 16384 data: 5 6 6 6
74482023-12-10T18:52:48.304Z07/30 Read block 39 len 16384
74492023-12-10T18:52:48.304Z08/30 Flush
74502023-12-10T18:52:48.304Z09/30 Write block 448 len 24576 data: 3 3 3 3 3 4
74512023-12-10T18:52:48.305Z10/30 Flush
74522023-12-10T18:52:48.305Z11/30 Write block 334 len 4096 data: 4
74532023-12-10T18:52:48.305Z12/30 Read block 575 len 24576
74542023-12-10T18:52:48.305Z13/30 Write block 54 len 4096 data: 4
74552023-12-10T18:52:48.306Z14/30 Read block 47 len 8192
74562023-12-10T18:52:48.306Z15/30 Read block 360 len 32768
74572023-12-10T18:52:48.306Z16/30 Read block 449 len 8192
74582023-12-10T18:52:48.306Z17/30 Read block 590 len 32768
74592023-12-10T18:52:48.307Z18/30 Write block 248 len 28672 data: 7 10 8 8 7 8 5
74602023-12-10T18:52:48.307Z19/30 Write block 197 len 28672 data: 7 6 6 5 5 5 6
74612023-12-10T18:52:48.307Z20/30 Write block 331 len 28672 data: 5 7 7 5 3 3 4
74622023-12-10T18:52:48.307Z21/30 Read block 532 len 28672
74632023-12-10T18:52:48.307Z22/30 Read block 505 len 4096
74642023-12-10T18:52:48.308Z23/30 Write block 561 len 36864 data: 5 5 5 5 5 5 4 4 4
74652023-12-10T18:52:48.308Z24/30 Write block 538 len 28672 data: 5 5 8 10 11 10 9
74662023-12-10T18:52:48.308Z25/30 Write block 289 len 4096 data: 5
74672023-12-10T18:52:48.308Z26/30 Flush
74682023-12-10T18:52:48.319Z27/30 Write block 569 len 24576 data: 5 5 4 3 4 6
74692023-12-10T18:52:48.731Z28/30 Write block 510 len 40960 data: 5 5 4 5 5 5 6 6 5 6
74702023-12-10T18:52:48.731Z29/30 Write block 575 len 12288 data: 7 7 7
74712023-12-10T18:52:48.732Z30/30 Write block 32 len 20480 data: 6 4 4 5 7
74722023-12-10T18:52:48.732Z----------------------------------------------------------------
74732023-12-10T18:52:48.732Z Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:29
74742023-12-10T18:52:48.734ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
74752023-12-10T18:52:48.734Z 2 Acked 1001 Write 9 Done Done Done false
74762023-12-10T18:52:48.735Z 3 Acked 1002 Read 8 Done Done Done false
74772023-12-10T18:52:48.736Z 4 Acked 1003 Read 5 Done Done Done false
74782023-12-10T18:52:48.736Z 5 Acked 1004 Read 7 Sent Done Done false
74792023-12-10T18:52:48.736Z 6 Acked 1005 Write 4 Sent Done Done false
74802023-12-10T18:52:48.736Z 7 Acked 1006 Read 4 Sent Done Done false
74812023-12-10T18:52:48.736Z 8 Acked 1007 Flush 0 Sent Done Done false
74822023-12-10T18:52:48.736Z 9 Acked 1008 Write 6 Sent Done Done false
74832023-12-10T18:52:48.737Z 10 Acked 1009 Flush 0 Sent Done Done false
74842023-12-10T18:52:48.737Z 11 Acked 1010 Write 1 Sent Done Done false
74852023-12-10T18:52:48.737Z 12 Acked 1011 Read 6 Sent Done Done false
74862023-12-10T18:52:48.737Z 13 Acked 1012 Write 1 Sent Done Done false
74872023-12-10T18:52:48.738Z 14 Acked 1013 Read 2 Sent Done Done false
74882023-12-10T18:52:48.738Z 15 Acked 1014 Read 8 Sent Done Done false
74892023-12-10T18:52:48.738Z 16 Acked 1015 Read 2 Sent Done Done false
74902023-12-10T18:52:48.738Z 17 Acked 1016 Read 8 Sent Done Done false
74912023-12-10T18:52:48.738Z 18 Acked 1017 Write 7 Sent Done Done false
74922023-12-10T18:52:48.739Z 19 Acked 1018 Write 7 Sent Done Done false
74932023-12-10T18:52:48.739Z 20 Acked 1019 Write 7 Sent Done Done false
74942023-12-10T18:52:48.739Z 21 Acked 1020 Read 7 Sent Done Done false
74952023-12-10T18:52:48.739Z 22 Acked 1021 Read 1 Sent Done Done false
74962023-12-10T18:52:48.739Z 23 Acked 1022 Write 9 Sent Done Done false
74972023-12-10T18:52:48.740Z 24 Acked 1023 Write 7 Sent Done Done false
74982023-12-10T18:52:48.740Z 25 Acked 1024 Write 1 Sent Done Done false
74992023-12-10T18:52:48.740Z 26 Acked 1025 Flush 0 Sent Done Done false
75002023-12-10T18:52:48.740Z 27 Acked 1026 Write 6 Sent Done Done false
75012023-12-10T18:52:48.741Z 28 Acked 1027 Write 10 Sent Done Done false
75022023-12-10T18:52:48.741Z 29 Acked 1028 Write 3 Sent Sent Sent false
75032023-12-10T18:52:48.741Z 30 Acked 1029 Write 5 Sent Sent Sent false
75042023-12-10T18:52:48.742Z STATES DS:0 DS:1 DS:2 TOTAL
75052023-12-10T18:52:48.742Z New 0 0 0 0
75062023-12-10T18:52:48.742Z Sent 26 2 2 30
75072023-12-10T18:52:48.742Z Done 3 27 27 57
75082023-12-10T18:52:48.743Z Skipped 0 0 0 0
75092023-12-10T18:52:48.743Z Error 0 0 0 0
75102023-12-10T18:52:48.743ZLast Flush: 1000 1025 1025
75112023-12-10T18:52:48.743ZDownstairs last five completed: 1000
75122023-12-10T18:52:48.743ZUpstairs last five completed: 30 29 28 27 26 25
75132023-12-10T18:52:48.744ZWrote out file "/tmp/test_repair_verify.data"
75142023-12-10T18:52:49.180Z
75152023-12-10T18:52:49.180ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
75162023-12-10T18:52:49.180ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
75172023-12-10T18:52:49.180Z 0 000-019 58 58 58 95 95 95 F F F
75182023-12-10T18:52:49.180Z 1 020-039 58 58 58 97 97 97 F T T <---
75192023-12-10T18:52:49.180Z 2 040-059 58 60 60 97 102 102 F F F <---
75202023-12-10T18:52:49.181Z 3 060-079 54 54 54 88 88 88 F F F
75212023-12-10T18:52:49.181Z 4 080-099 57 57 57 94 94 94 F F F
75222023-12-10T18:52:49.181Z 5 100-119 58 58 58 95 95 95 F F F
75232023-12-10T18:52:49.181Z 6 120-139 59 59 59 98 98 98 F F F
75242023-12-10T18:52:49.181Z 7 140-159 59 59 59 98 98 98 F F F
75252023-12-10T18:52:49.181Z 8 160-179 54 54 54 90 90 90 F F F
75262023-12-10T18:52:49.181Z 9 180-199 48 60 60 81 102 102 T F F <---
75272023-12-10T18:52:49.181Z 10 200-219 56 60 60 93 102 102 F F F <---
75282023-12-10T18:52:49.182Z 11 220-239 56 56 56 93 93 93 F F F
75292023-12-10T18:52:49.182Z 12 240-259 51 60 60 84 102 102 F F F <---
75302023-12-10T18:52:49.182Z 13 260-279 55 55 55 92 92 92 F F F
75312023-12-10T18:52:49.182Z 14 280-299 59 60 60 98 102 102 F F F <---
75322023-12-10T18:52:49.182Z 15 300-319 59 59 59 98 98 98 F F F
75332023-12-10T18:52:49.182Z 16 320-339 59 60 60 98 102 102 F F F <---
75342023-12-10T18:52:49.183Z 17 340-359 58 58 58 96 96 96 F F F
75352023-12-10T18:52:49.183Z 18 360-379 59 59 59 98 98 98 F F F
75362023-12-10T18:52:49.183Z 19 380-399 59 59 59 98 98 98 F F F
75372023-12-10T18:52:49.185Z 20 400-419 49 49 49 82 82 82 F F F
75382023-12-10T18:52:49.185Z 21 420-439 59 59 59 98 98 98 F F F
75392023-12-10T18:52:49.185Z 22 440-459 59 60 60 98 101 101 F F F <---
75402023-12-10T18:52:49.185Z 23 460-479 58 58 58 96 96 96 F F F
75412023-12-10T18:52:49.186Z 24 480-499 59 59 59 98 98 98 F F F
75422023-12-10T18:52:49.186Z 25 500-519 59 59 59 98 98 98 F T T <---
75432023-12-10T18:52:49.186Z 26 520-539 57 60 60 94 102 102 F F F <---
75442023-12-10T18:52:49.186Z 27 540-559 59 60 60 98 102 102 F F F <---
75452023-12-10T18:52:49.186Z 28 560-579 59 60 60 98 102 102 F T T <---
75462023-12-10T18:52:49.186Z 29 580-599 50 50 50 83 83 83 F F F
75472023-12-10T18:52:49.187ZMax gen: 60, Max flush: 102
75482023-12-10T18:52:49.187ZError: Difference in extent metadata found!
75492023-12-10T18:52:49.187ZA Difference in extent metadata is expected here
75502023-12-10T18:52:49.187ZOn loop 30
75512023-12-10T18:52:49.187Z
75522023-12-10T18:52:49.187ZVerifying data now
75532023-12-10T18:52:53.552ZNo change after verify
75542023-12-10T18:52:53.552ZLoop: 30 Downstairs dump after verify (and repair):
75552023-12-10T18:52:53.583ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
75562023-12-10T18:52:53.583Z 0 000-019 58 58 58 95 95 95 F F F
75572023-12-10T18:52:53.601Z 1 020-039 61 61 61 103 103 103 F F F
75582023-12-10T18:52:53.601Z 2 040-059 60 60 60 102 102 102 F F F
75592023-12-10T18:52:53.602Z 3 060-079 54 54 54 88 88 88 F F F
75602023-12-10T18:52:53.602Z 4 080-099 57 57 57 94 94 94 F F F
75612023-12-10T18:52:53.602Z 5 100-119 58 58 58 95 95 95 F F F
75622023-12-10T18:52:53.602Z 6 120-139 59 59 59 98 98 98 F F F
75632023-12-10T18:52:53.602Z 7 140-159 59 59 59 98 98 98 F F F
75642023-12-10T18:52:53.602Z 8 160-179 54 54 54 90 90 90 F F F
75652023-12-10T18:52:53.603Z 9 180-199 60 60 60 102 102 102 F F F
75662023-12-10T18:52:53.603Z 10 200-219 60 60 60 102 102 102 F F F
75672023-12-10T18:52:53.603Z 11 220-239 56 56 56 93 93 93 F F F
75682023-12-10T18:52:53.603Z 12 240-259 60 60 60 102 102 102 F F F
75692023-12-10T18:52:53.603Z 13 260-279 55 55 55 92 92 92 F F F
75702023-12-10T18:52:53.604Z 14 280-299 60 60 60 102 102 102 F F F
75712023-12-10T18:52:53.604Z 15 300-319 59 59 59 98 98 98 F F F
75722023-12-10T18:52:53.604Z 16 320-339 60 60 60 102 102 102 F F F
75732023-12-10T18:52:53.604Z 17 340-359 58 58 58 96 96 96 F F F
75742023-12-10T18:52:53.605Z 18 360-379 59 59 59 98 98 98 F F F
75752023-12-10T18:52:53.605Z 19 380-399 59 59 59 98 98 98 F F F
75762023-12-10T18:52:53.605Z 20 400-419 49 49 49 82 82 82 F F F
75772023-12-10T18:52:53.605Z 21 420-439 59 59 59 98 98 98 F F F
75782023-12-10T18:52:53.605Z 22 440-459 60 60 60 101 101 101 F F F
75792023-12-10T18:52:53.605Z 23 460-479 58 58 58 96 96 96 F F F
75802023-12-10T18:52:53.606Z 24 480-499 59 59 59 98 98 98 F F F
75812023-12-10T18:52:53.606Z 25 500-519 61 61 61 103 103 103 F F F
75822023-12-10T18:52:53.606Z 26 520-539 60 60 60 102 102 102 F F F
75832023-12-10T18:52:53.606Z 27 540-559 60 60 60 102 102 102 F F F
75842023-12-10T18:52:53.606Z 28 560-579 61 61 61 103 103 103 F F F
75852023-12-10T18:52:53.607Z 29 580-599 50 50 50 83 83 83 F F F
75862023-12-10T18:52:53.607ZMax gen: 61, Max flush: 103
75872023-12-10T18:52:53.607Z
75882023-12-10T18:52:53.607ZBegin loop 31
75892023-12-10T18:52:53.607ZDownstairs to restart: 1
75902023-12-10T18:52:54.059Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.604796363Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410}
75912023-12-10T18:52:54.060Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.60536062Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410}
75922023-12-10T18:52:54.060Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.605446545Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410}
75932023-12-10T18:52:54.060Z{"msg":"Crucible 32a72908-bc5e-40b0-a7d0-3948b900297d has session id: 090d9375-4608-4937-8746-3e84fbd00f2f","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.605473714Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
75942023-12-10T18:52:54.060Z{"msg":"Upstairs opts: Upstairs UUID: 32a72908-bc5e-40b0-a7d0-3948b900297d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.605517873Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
75952023-12-10T18:52:54.061Z{"msg":"Crucible stats registered with UUID: 32a72908-bc5e-40b0-a7d0-3948b900297d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.60559222Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
75962023-12-10T18:52:54.061ZCrucible runtime is spawned
75972023-12-10T18:52:54.061ZThe guest has requested activation with gen:62
75982023-12-10T18:52:54.061Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.605957898Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f","task":"up_listen"}
75992023-12-10T18:52:54.061Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.606051125Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76002023-12-10T18:52:54.061Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.606089927Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76012023-12-10T18:52:54.062Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.606134946Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"looper":"2","session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76022023-12-10T18:52:54.062Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.606205741Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"looper":"1","session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76032023-12-10T18:52:54.062Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.606275062Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"looper":"0","session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76042023-12-10T18:52:54.062Z{"msg":"32a72908-bc5e-40b0-a7d0-3948b900297d active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.60655556Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76052023-12-10T18:52:54.062Z{"msg":"[2] 32a72908-bc5e-40b0-a7d0-3948b900297d looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.606613793Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"looper":"2","session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76062023-12-10T18:52:54.062Z{"msg":"Set desired generation to :62","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.606652752Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76072023-12-10T18:52:54.062Z{"msg":"[0] 32a72908-bc5e-40b0-a7d0-3948b900297d looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.606699968Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"looper":"0","session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76082023-12-10T18:52:54.062Z{"msg":"[1] 32a72908-bc5e-40b0-a7d0-3948b900297d looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.606730941Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"looper":"1","session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76092023-12-10T18:52:54.062Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.606797039Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76102023-12-10T18:52:54.062Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.607062383Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76112023-12-10T18:52:54.062Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.607171177Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76122023-12-10T18:52:54.062Z{"msg":"[2] 32a72908-bc5e-40b0-a7d0-3948b900297d (090d9375-4608-4937-8746-3e84fbd00f2f) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.607411146Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76132023-12-10T18:52:54.062Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.607555482Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76142023-12-10T18:52:54.062Z{"msg":"[2] client is_active_req TRUE, promote! session 090d9375-4608-4937-8746-3e84fbd00f2f","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.607632285Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76152023-12-10T18:52:54.063Z{"msg":"[0] 32a72908-bc5e-40b0-a7d0-3948b900297d (090d9375-4608-4937-8746-3e84fbd00f2f) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.607701517Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76162023-12-10T18:52:54.063Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.607744062Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76172023-12-10T18:52:54.063Z{"msg":"[0] client is_active_req TRUE, promote! session 090d9375-4608-4937-8746-3e84fbd00f2f","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.607824354Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76182023-12-10T18:52:54.063Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.608287806Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76192023-12-10T18:52:54.063Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.608371436Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76202023-12-10T18:52:54.063Z{"msg":"32a72908-bc5e-40b0-a7d0-3948b900297d WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.608399204Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76212023-12-10T18:52:54.063Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.608445305Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76222023-12-10T18:52:54.063Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.608538856Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76232023-12-10T18:52:54.063Z{"msg":"32a72908-bc5e-40b0-a7d0-3948b900297d WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.608633617Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76242023-12-10T18:52:54.063Z{"msg":"[1] 32a72908-bc5e-40b0-a7d0-3948b900297d (090d9375-4608-4937-8746-3e84fbd00f2f) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.60896193Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76252023-12-10T18:52:54.063Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.609056199Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76262023-12-10T18:52:54.063Z{"msg":"[1] client is_active_req TRUE, promote! session 090d9375-4608-4937-8746-3e84fbd00f2f","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.609140822Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76272023-12-10T18:52:54.063Z{"msg":"[2] 32a72908-bc5e-40b0-a7d0-3948b900297d (090d9375-4608-4937-8746-3e84fbd00f2f) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.609216471Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76282023-12-10T18:52:54.063Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.609257425Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76292023-12-10T18:52:54.063Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:53.609293921Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76302023-12-10T18:52:54.063Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.609360029Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76312023-12-10T18:52:54.063Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.60940166Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76322023-12-10T18:52:54.063Z{"msg":"32a72908-bc5e-40b0-a7d0-3948b900297d WaitActive WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.609466911Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76332023-12-10T18:52:54.064Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.609492847Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76342023-12-10T18:52:54.064Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.609548964Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76352023-12-10T18:52:54.064Z{"msg":"[0] 32a72908-bc5e-40b0-a7d0-3948b900297d (090d9375-4608-4937-8746-3e84fbd00f2f) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.609625901Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76362023-12-10T18:52:54.064Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.609699673Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76372023-12-10T18:52:54.064Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:53.609753388Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76382023-12-10T18:52:54.064Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.609786283Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76392023-12-10T18:52:54.064Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.609855851Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76402023-12-10T18:52:54.064Z{"msg":"32a72908-bc5e-40b0-a7d0-3948b900297d WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.60993206Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76412023-12-10T18:52:54.064Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.610003669Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76422023-12-10T18:52:54.064Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.610072077Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76432023-12-10T18:52:54.065Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.610143845Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76442023-12-10T18:52:54.065Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.610220467Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76452023-12-10T18:52:54.065Z{"msg":"32a72908-bc5e-40b0-a7d0-3948b900297d WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.610293918Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76462023-12-10T18:52:54.065Z{"msg":"[1] 32a72908-bc5e-40b0-a7d0-3948b900297d (090d9375-4608-4937-8746-3e84fbd00f2f) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.610553693Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76472023-12-10T18:52:54.065Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.610640368Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76482023-12-10T18:52:54.065Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:52:53.610714308Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76492023-12-10T18:52:54.065Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.61075335Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76502023-12-10T18:52:54.065Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.610818079Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76512023-12-10T18:52:54.066Z{"msg":"32a72908-bc5e-40b0-a7d0-3948b900297d WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.610857488Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76522023-12-10T18:52:54.066Z{"msg":"[0]R flush_numbers[0..12]: [95, 103, 102, 88, 94, 95, 98, 98, 90, 102, 102, 93]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:62
76532023-12-10T18:52:54.068Z,"time":"2023-12-10T18:52:53.610923529Z","hostname":"Wait for a query_work_queue command to finish before sending IO
76542023-12-10T18:52:54.068Zip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76552023-12-10T18:52:54.068Z{"msg":"[0]R generation[0..12]: [58, 61, 60, 54, 57, 58, 59, 59, 54, 60, 60, 56]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.611004786Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76562023-12-10T18:52:54.068Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.611079145Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76572023-12-10T18:52:54.068Z{"msg":"[1]R flush_numbers[0..12]: [95, 103, 102, 88, 94, 95, 98, 98, 90, 102, 102, 93]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.611150257Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76582023-12-10T18:52:54.068Z{"msg":"[1]R generation[0..12]: [58, 61, 60, 54, 57, 58, 59, 59, 54, 60, 60, 56]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.611223934Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76592023-12-10T18:52:54.068Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.611268987Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76602023-12-10T18:52:54.068Z{"msg":"[2]R flush_numbers[0..12]: [95, 103, 102, 88, 94, 95, 98, 98, 90, 102, 102, 93]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.611342652Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76612023-12-10T18:52:54.068Z{"msg":"[2]R generation[0..12]: [58, 61, 60, 54, 57, 58, 59, 59, 54, 60, 60, 56]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.611383989Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76622023-12-10T18:52:54.068Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.611474556Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76632023-12-10T18:52:54.068Z{"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.611514468Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76642023-12-10T18:52:54.069Z{"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.611574119Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76652023-12-10T18:52:54.069Z{"msg":"Next flush: 104","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.611613018Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76662023-12-10T18:52:54.069Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.611667682Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76672023-12-10T18:52:54.070Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.611707048Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76682023-12-10T18:52:54.070Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.611767989Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76692023-12-10T18:52:54.070Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.611806898Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76702023-12-10T18:52:54.070Z{"msg":"32a72908-bc5e-40b0-a7d0-3948b900297d is now active with session: 090d9375-4608-4937-8746-3e84fbd00f2f","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.611865195Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76712023-12-10T18:52:54.070Z{"msg":"32a72908-bc5e-40b0-a7d0-3948b900297d Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.611905471Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76722023-12-10T18:52:54.070Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
76732023-12-10T18:52:54.070Z,"time":"2023-12-10T18:52:53.612009377Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76742023-12-10T18:52:54.070Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.612107375Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76752023-12-10T18:52:54.070Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.612174182Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76762023-12-10T18:52:54.070Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.612212204Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76772023-12-10T18:52:54.070Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.612267212Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76782023-12-10T18:52:54.070Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.612306028Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76792023-12-10T18:52:54.070Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.61235851Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76802023-12-10T18:52:54.070Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.612396391Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76812023-12-10T18:52:54.070Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.612450175Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76822023-12-10T18:52:54.070Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:52:53.612488409Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76832023-12-10T18:52:54.071Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30Loading write count information from file "/tmp/test_repair_verify.data"
76842023-12-10T18:52:54.071Z,"time":"2023-12-10T18:52:53.612524925Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1410Run Repair workload
76852023-12-10T18:52:54.071Z,"session_id":"090d9375-4608-4937-8746-3e84fbd00f2f"}
76862023-12-10T18:52:54.071Z01/30 Write block 495 len 36864 data: 4 3 3 3 4 7 7 7 8
76872023-12-10T18:52:54.071Z02/30 Read block 301 len 32768
76882023-12-10T18:52:54.071Z03/30 Read block 101 len 20480
76892023-12-10T18:52:54.071Z04/30 Write block 299 len 40960 data: 7 5 5 6 8 9 6 6 4 4
76902023-12-10T18:52:54.077Z05/30 Write block 174 len 36864 data: 7 5 5 6 4 2 2 4 5
76912023-12-10T18:52:54.077Z06/30 Read block 390 len 24576
76922023-12-10T18:52:54.077Z07/30 Read block 560 len 20480
76932023-12-10T18:52:54.077Z08/30 Read block 273 len 36864
76942023-12-10T18:52:54.077Z09/30 Read block 403 len 32768
76952023-12-10T18:52:54.077Z10/30 Read block 339 len 16384
76962023-12-10T18:52:54.077Z11/30 Read block 262 len 8192
76972023-12-10T18:52:54.077Z12/30 Write block 506 len 36864 data: 6 6 6 5 6 6 5 6 6
76982023-12-10T18:52:54.077Z13/30 Write block 13 len 8192 data: 7 7
76992023-12-10T18:52:54.077Z14/30 Read block 236 len 24576
77002023-12-10T18:52:54.077Z15/30 Flush
77012023-12-10T18:52:54.077Z16/30 Write block 183 len 32768 data: 6 4 4 3 4 5 5 5
77022023-12-10T18:52:54.078Z17/30 Read block 11 len 28672
77032023-12-10T18:52:54.078Z18/30 Write block 564 len 32768 data: 6 6 6 5 5 6 6 5
77042023-12-10T18:52:54.078Z19/30 Write block 586 len 12288 data: 2 3 3
77052023-12-10T18:52:54.078Z20/30 Write block 301 len 16384 data: 6 7 9 10
77062023-12-10T18:52:54.078Z21/30 Write block 334 len 40960 data: 6 4 4 5 4 4 3 4 6 7
77072023-12-10T18:52:54.078Z22/30 Read block 570 len 36864
77082023-12-10T18:52:54.078Z23/30 Flush
77092023-12-10T18:52:54.078Z24/30 Write block 299 len 12288 data: 8 6 7
77102023-12-10T18:52:54.078Z25/30 Write block 380 len 40960 data: 10 7 8 8 9 9 8 8 7 6
77112023-12-10T18:52:54.078Z26/30 Write block 108 len 12288 data: 6 5 5
77122023-12-10T18:52:54.078Z27/30 Read block 217 len 16384
77132023-12-10T18:52:54.078Z28/30 Read block 88 len 8192
77142023-12-10T18:52:54.079Z29/30 Read block 425 len 32768
77152023-12-10T18:52:54.079Z30/30 Write block 404 len 32768 data: 6 6 5 6 6 4 6 4
77162023-12-10T18:52:54.079Z----------------------------------------------------------------
77172023-12-10T18:52:54.079Z Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:30
77182023-12-10T18:52:54.079ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
77192023-12-10T18:52:54.079Z 1 Acked 1000 Write 9 Done Done Done false
77202023-12-10T18:52:54.079Z 2 Acked 1001 Read 8 Done Done Done false
77212023-12-10T18:52:54.079Z 3 Acked 1002 Read 5 Done Done Done false
77222023-12-10T18:52:54.079Z 4 Acked 1003 Write 10 Done Done Done false
77232023-12-10T18:52:54.080Z 5 Acked 1004 Write 9 Done Sent Done false
77242023-12-10T18:52:54.080Z 6 Acked 1005 Read 6 Done Sent Done false
77252023-12-10T18:52:54.080Z 7 Acked 1006 Read 5 Done Sent Done false
77262023-12-10T18:52:54.080Z 8 Acked 1007 Read 9 Done Sent Done false
77272023-12-10T18:52:54.080Z 9 Acked 1008 Read 8 Done Sent Done false
77282023-12-10T18:52:54.080Z 10 Acked 1009 Read 4 Done Sent Done false
77292023-12-10T18:52:54.080Z 11 Acked 1010 Read 2 Done Sent Done false
77302023-12-10T18:52:54.080Z 12 Acked 1011 Write 9 Done Sent Done false
77312023-12-10T18:52:54.080Z 13 Acked 1012 Write 2 Done Sent Done false
77322023-12-10T18:52:54.080Z 14 Acked 1013 Read 6 Done Sent Done false
77332023-12-10T18:52:54.080Z 15 Acked 1014 Flush 0 Done Sent Done false
77342023-12-10T18:52:54.080Z 16 Acked 1015 Write 8 Done Sent Done false
77352023-12-10T18:52:54.080Z 17 Acked 1016 Read 7 Done Sent Done false
77362023-12-10T18:52:54.080Z 18 Acked 1017 Write 8 Done Sent Done false
77372023-12-10T18:52:54.080Z 19 Acked 1018 Write 3 Done Sent Done false
77382023-12-10T18:52:54.080Z 20 Acked 1019 Write 4 Done Sent Done false
77392023-12-10T18:52:54.080Z 21 Acked 1020 Write 10 Done Sent Done false
77402023-12-10T18:52:54.081Z 22 Acked 1021 Read 9 Done Sent Done false
77412023-12-10T18:52:54.081Z 23 Acked 1022 Flush 0 Done Sent Done false
77422023-12-10T18:52:54.081Z 24 Acked 1023 Write 3 Done Sent Done false
77432023-12-10T18:52:54.081Z 25 Acked 1024 Write 10 Done Sent Done false
77442023-12-10T18:52:54.081Z 26 Acked 1025 Write 3 Done Sent Done false
77452023-12-10T18:52:54.081Z 27 Acked 1026 Read 4 Done Sent Done false
77462023-12-10T18:52:54.081Z 28 Acked 1027 Read 2 Done Sent Done false
77472023-12-10T18:52:54.081Z 29 Acked 1028 Read 8 Done Sent Done false
77482023-12-10T18:52:54.081Z 30 Acked 1029 Write 8 Sent Sent Sent false
77492023-12-10T18:52:54.081Z STATES DS:0 DS:1 DS:2 TOTAL
77502023-12-10T18:52:54.081Z New 0 0 0 0
77512023-12-10T18:52:54.081Z Sent 1 26 1 28
77522023-12-10T18:52:54.081Z Done 29 4 29 62
77532023-12-10T18:52:54.081Z Skipped 0 0 0 0
77542023-12-10T18:52:54.081Z Error 0 0 0 0
77552023-12-10T18:52:54.081ZLast Flush: 1022 0 1022
77562023-12-10T18:52:54.082ZDownstairs last five completed:
77572023-12-10T18:52:54.082ZUpstairs last five completed: 30 29 28 27 26 25
77582023-12-10T18:52:54.082ZWrote out file "/tmp/test_repair_verify.data"
77592023-12-10T18:52:54.082Z
77602023-12-10T18:52:54.082ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
77612023-12-10T18:52:54.082ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
77622023-12-10T18:52:54.082Z 0 000-019 62 58 62 104 95 104 F F F <---
77632023-12-10T18:52:54.082Z 1 020-039 61 61 61 103 103 103 F F F
77642023-12-10T18:52:54.082Z 2 040-059 60 60 60 102 102 102 F F F
77652023-12-10T18:52:54.082Z 3 060-079 54 54 54 88 88 88 F F F
77662023-12-10T18:52:54.082Z 4 080-099 57 57 57 94 94 94 F F F
77672023-12-10T18:52:54.082Z 5 100-119 58 58 58 95 95 95 T F T <---
77682023-12-10T18:52:54.082Z 6 120-139 59 59 59 98 98 98 F F F
77692023-12-10T18:52:54.082Z 7 140-159 59 59 59 98 98 98 F F F
77702023-12-10T18:52:54.082Z 8 160-179 62 54 62 104 90 104 F F F <---
77712023-12-10T18:52:54.083Z 9 180-199 62 60 62 105 102 105 F F F <---
77722023-12-10T18:52:54.083Z 10 200-219 60 60 60 102 102 102 F F F
77732023-12-10T18:52:54.083Z 11 220-239 56 56 56 93 93 93 F F F
77742023-12-10T18:52:54.083Z 12 240-259 60 60 60 102 102 102 F F F
77752023-12-10T18:52:54.083Z 13 260-279 55 55 55 92 92 92 F F F
77762023-12-10T18:52:54.083Z 14 280-299 62 60 62 104 102 104 T T T <---
77772023-12-10T18:52:54.083Z 15 300-319 62 59 62 105 98 105 T T T <---
77782023-12-10T18:52:54.083Z 16 320-339 62 60 62 105 102 105 F F F <---
77792023-12-10T18:52:54.083Z 17 340-359 62 58 62 105 96 105 F F F <---
77802023-12-10T18:52:54.083Z 18 360-379 59 59 59 98 98 98 F F F
77812023-12-10T18:52:54.083Z 19 380-399 59 59 59 98 98 98 T F T <---
77822023-12-10T18:52:54.083Z 20 400-419 49 49 49 82 82 82 T F T <---
77832023-12-10T18:52:54.083Z 21 420-439 59 59 59 98 98 98 F F F
77842023-12-10T18:52:54.083Z 22 440-459 60 60 60 101 101 101 F F F
77852023-12-10T18:52:54.084Z 23 460-479 58 58 58 96 96 96 F F F
77862023-12-10T18:52:54.084Z 24 480-499 62 59 62 104 98 104 F T F <---
77872023-12-10T18:52:54.084Z 25 500-519 62 61 62 104 103 104 F T F <---
77882023-12-10T18:52:54.084Z 26 520-539 60 60 60 102 102 102 F F F
77892023-12-10T18:52:54.084Z 27 540-559 60 60 60 102 102 102 F F F
77902023-12-10T18:52:54.100Z 28 560-579 62 61 62 105 103 105 F F F <---
77912023-12-10T18:52:54.100Z 29 580-599 62 50 62 105 83 105 F F F <---
77922023-12-10T18:52:54.100ZMax gen: 62, Max flush: 105
77932023-12-10T18:52:54.100ZError: Difference in extent metadata found!
77942023-12-10T18:52:54.100ZA Difference in extent metadata is expected here
77952023-12-10T18:52:54.100ZOn loop 31
77962023-12-10T18:52:54.100Z
77972023-12-10T18:52:54.100ZVerifying data now
77982023-12-10T18:53:00.061ZNo change after verify
77992023-12-10T18:53:00.061ZLoop: 31 Downstairs dump after verify (and repair):
78002023-12-10T18:53:00.089ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
78012023-12-10T18:53:00.089Z 0 000-019 62 62 62 104 104 104 F F F
78022023-12-10T18:53:00.089Z 1 020-039 61 61 61 103 103 103 F F F
78032023-12-10T18:53:00.089Z 2 040-059 60 60 60 102 102 102 F F F
78042023-12-10T18:53:00.089Z 3 060-079 54 54 54 88 88 88 F F F
78052023-12-10T18:53:00.090Z 4 080-099 57 57 57 94 94 94 F F F
78062023-12-10T18:53:00.090Z 5 100-119 63 63 63 106 106 106 F F F
78072023-12-10T18:53:00.090Z 6 120-139 59 59 59 98 98 98 F F F
78082023-12-10T18:53:00.090Z 7 140-159 59 59 59 98 98 98 F F F
78092023-12-10T18:53:00.090Z 8 160-179 62 62 62 104 104 104 F F F
78102023-12-10T18:53:00.090Z 9 180-199 62 62 62 105 105 105 F F F
78112023-12-10T18:53:00.107Z 10 200-219 60 60 60 102 102 102 F F F
78122023-12-10T18:53:00.108Z 11 220-239 56 56 56 93 93 93 F F F
78132023-12-10T18:53:00.108Z 12 240-259 60 60 60 102 102 102 F F F
78142023-12-10T18:53:00.108Z 13 260-279 55 55 55 92 92 92 F F F
78152023-12-10T18:53:00.108Z 14 280-299 63 63 63 106 106 106 F F F
78162023-12-10T18:53:00.108Z 15 300-319 63 63 63 106 106 106 F F F
78172023-12-10T18:53:00.108Z 16 320-339 62 62 62 105 105 105 F F F
78182023-12-10T18:53:00.108Z 17 340-359 62 62 62 105 105 105 F F F
78192023-12-10T18:53:00.108Z 18 360-379 59 59 59 98 98 98 F F F
78202023-12-10T18:53:00.108Z 19 380-399 63 63 63 106 106 106 F F F
78212023-12-10T18:53:00.108Z 20 400-419 63 63 63 106 106 106 F F F
78222023-12-10T18:53:00.108Z 21 420-439 59 59 59 98 98 98 F F F
78232023-12-10T18:53:00.109Z 22 440-459 60 60 60 101 101 101 F F F
78242023-12-10T18:53:00.109Z 23 460-479 58 58 58 96 96 96 F F F
78252023-12-10T18:53:00.109Z 24 480-499 62 62 62 104 104 104 F F F
78262023-12-10T18:53:00.109Z 25 500-519 62 62 62 104 104 104 F F F
78272023-12-10T18:53:00.109Z 26 520-539 60 60 60 102 102 102 F F F
78282023-12-10T18:53:00.109Z 27 540-559 60 60 60 102 102 102 F F F
78292023-12-10T18:53:00.109Z 28 560-579 62 62 62 105 105 105 F F F
78302023-12-10T18:53:00.109Z 29 580-599 62 62 62 105 105 105 F F F
78312023-12-10T18:53:00.109ZMax gen: 63, Max flush: 106
78322023-12-10T18:53:00.109Z
78332023-12-10T18:53:00.109ZBegin loop 32
78342023-12-10T18:53:00.109ZDownstairs to restart: 2
78352023-12-10T18:53:00.122Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.124165567Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418}
78362023-12-10T18:53:00.122Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.124607621Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418}
78372023-12-10T18:53:00.128Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.124674564Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418}
78382023-12-10T18:53:00.128Z{"msg":"Crucible 96861fd5-6e4f-444f-9836-fda5fac7d85b has session id: e1856594-b734-4d14-81bb-0e652ea15c03","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.124746623Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78392023-12-10T18:53:00.128Z{"msg":"Upstairs opts: Upstairs UUID: 96861fd5-6e4f-444f-9836-fda5fac7d85b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.124803714Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78402023-12-10T18:53:00.128Z{"msg":"Crucible stats registered with UUID: 96861fd5-6e4f-444f-9836-fda5fac7d85b","v":Crucible runtime is spawned
78412023-12-10T18:53:00.128Z0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.124948593Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78422023-12-10T18:53:00.128Z{"msg":"up_listen starts","v"The guest has requested activation with gen:64
78432023-12-10T18:53:00.128Z:0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.125082875Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03","task":"up_listen"}
78442023-12-10T18:53:00.128Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.125153024Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78452023-12-10T18:53:00.128Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.125193106Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"looper":"1","session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78462023-12-10T18:53:00.128Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.125297873Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78472023-12-10T18:53:00.128Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.125337153Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"looper":"0","session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78482023-12-10T18:53:00.129Z{"msg":"[1] 96861fd5-6e4f-444f-9836-fda5fac7d85b looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.125466084Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"looper":"1","session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78492023-12-10T18:53:00.129Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.125692041Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"looper":"2","session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78502023-12-10T18:53:00.129Z{"msg":"[0] 96861fd5-6e4f-444f-9836-fda5fac7d85b looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.1257382Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"looper":"0","session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78512023-12-10T18:53:00.129Z{"msg":"[2] 96861fd5-6e4f-444f-9836-fda5fac7d85b looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.125805032Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"looper":"2","session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78522023-12-10T18:53:00.129Z{"msg":"96861fd5-6e4f-444f-9836-fda5fac7d85b active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.126036278Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78532023-12-10T18:53:00.129Z{"msg":"Set desired generation to :64","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.126099669Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78542023-12-10T18:53:00.129Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.1263159Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78552023-12-10T18:53:00.129Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.126500681Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78562023-12-10T18:53:00.129Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.126808596Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78572023-12-10T18:53:00.129Z{"msg":"[0] 96861fd5-6e4f-444f-9836-fda5fac7d85b (e1856594-b734-4d14-81bb-0e652ea15c03) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.127171116Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78582023-12-10T18:53:00.129Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.127309249Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78592023-12-10T18:53:00.129Z{"msg":"[0] client is_active_req TRUE, promote! session e1856594-b734-4d14-81bb-0e652ea15c03","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.127478651Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78602023-12-10T18:53:00.129Z{"msg":"[1] 96861fd5-6e4f-444f-9836-fda5fac7d85b (e1856594-b734-4d14-81bb-0e652ea15c03) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.127585169Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78612023-12-10T18:53:00.129Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.127625184Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78622023-12-10T18:53:00.129Z{"msg":"[1] client is_active_req TRUE, promote! session e1856594-b734-4d14-81bb-0e652ea15c03","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.127662495Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78632023-12-10T18:53:00.129Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.127914697Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78642023-12-10T18:53:00.130Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.1279993Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78652023-12-10T18:53:00.130Z{"msg":"96861fd5-6e4f-444f-9836-fda5fac7d85b WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.128075528Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78662023-12-10T18:53:00.130Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.128158643Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78672023-12-10T18:53:00.130Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.128224902Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78682023-12-10T18:53:00.130Z{"msg":"96861fd5-6e4f-444f-9836-fda5fac7d85b WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.12825116Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78692023-12-10T18:53:00.130Z{"msg":"[2] 96861fd5-6e4f-444f-9836-fda5fac7d85b (e1856594-b734-4d14-81bb-0e652ea15c03) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.12831194Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78702023-12-10T18:53:00.132Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.128383238Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78712023-12-10T18:53:00.132Z{"msg":"[2] client is_active_req TRUE, promote! session e1856594-b734-4d14-81bb-0e652ea15c03","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.128457859Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78722023-12-10T18:53:00.139Z{"msg":"[0] 96861fd5-6e4f-444f-9836-fda5fac7d85b (e1856594-b734-4d14-81bb-0e652ea15c03) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.128863304Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78732023-12-10T18:53:00.139Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.128960423Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78742023-12-10T18:53:00.139Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:00.129004863Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78752023-12-10T18:53:00.139Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.12906715Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78762023-12-10T18:53:00.139Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.129093289Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78772023-12-10T18:53:00.139Z{"msg":"96861fd5-6e4f-444f-9836-fda5fac7d85b WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.129199829Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78782023-12-10T18:53:00.139Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.129242323Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78792023-12-10T18:53:00.139Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.129310827Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78802023-12-10T18:53:00.139Z{"msg":"[1] 96861fd5-6e4f-444f-9836-fda5fac7d85b (e1856594-b734-4d14-81bb-0e652ea15c03) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.129356978Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78812023-12-10T18:53:00.139Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.12942075Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78822023-12-10T18:53:00.139Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:00.129447305Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78832023-12-10T18:53:00.139Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.129507412Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78842023-12-10T18:53:00.139Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.129603165Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78852023-12-10T18:53:00.139Z{"msg":"96861fd5-6e4f-444f-9836-fda5fac7d85b WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.129751658Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78862023-12-10T18:53:00.139Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.129795946Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78872023-12-10T18:53:00.140Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.129865823Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78882023-12-10T18:53:00.140Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.129938043Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78892023-12-10T18:53:00.140Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.130087621Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78902023-12-10T18:53:00.140Z{"msg":"96861fd5-6e4f-444f-9836-fda5fac7d85b WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.13017989Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78912023-12-10T18:53:00.140Z{"msg":"[2] 96861fd5-6e4f-444f-9836-fda5fac7d85b (e1856594-b734-4d14-81bb-0e652ea15c03) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.130254403Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78922023-12-10T18:53:00.140Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.130298686Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78932023-12-10T18:53:00.140Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:00.130415507Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78942023-12-10T18:53:00.140Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.130459131Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78952023-12-10T18:53:00.141Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.130491792Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78962023-12-10T18:53:00.141Z{"msg":"96861fd5-6e4f-444f-9836-fda5fac7d85b WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.130537793Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
78972023-12-10T18:53:00.141ZThe guest has finished waiting for activation with:64
78982023-12-10T18:53:00.141Z{"msg":"Wait for a query_work_queue command to finish before sending IO
78992023-12-10T18:53:00.141Z[0]R flush_numbers[0..12]: [104, 103, 102, 88, 94, 106, 98, 98, 104, 105, 102, 93]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.130606055Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79002023-12-10T18:53:00.141Z{"msg":"[0]R generation[0..12]: [62, 61, 60, 54, 57, 63, 59, 59, 62, 62, 60, 56]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.130631166Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79012023-12-10T18:53:00.141Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.130683791Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79022023-12-10T18:53:00.141Z{"msg":"[1]R flush_numbers[0..12]: [104, 103, 102, 88, 94, 106, 98, 98, 104, 105, 102, 93]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.130707782Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79032023-12-10T18:53:00.141Z{"msg":"[1]R generation[0..12]: [62, 61, 60, 54, 57, 63, 59, 59, 62, 62, 60, 56]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.130728802Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79042023-12-10T18:53:00.141Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.130800564Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79052023-12-10T18:53:00.141Z{"msg":"[2]R flush_numbers[0..12]: [104, 103, 102, 88, 94, 106, 98, 98, 104, 105, 102, 93]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.130842085Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79062023-12-10T18:53:00.141Z{"msg":"[2]R generation[0..12]: [62, 61, 60, 54, 57, 63, 59, 59, 62, 62, 60, 56]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.13090265Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79072023-12-10T18:53:00.141Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.130973469Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79082023-12-10T18:53:00.141Z{"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.131013369Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79092023-12-10T18:53:00.141Z{"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.131067912Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79102023-12-10T18:53:00.141Z{"msg":"Next flush: 107","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.131106915Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79112023-12-10T18:53:00.142Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.13117423Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79122023-12-10T18:53:00.142Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.131213466Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79132023-12-10T18:53:00.142Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.131284058Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79142023-12-10T18:53:00.142Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.131323262Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79152023-12-10T18:53:00.142Z{"msg":"96861fd5-6e4f-444f-9836-fda5fac7d85b is now active with session: e1856594-b734-4d14-81bb-0e652ea15c03","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.131382487Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79162023-12-10T18:53:00.142Z{"msg":"96861fd5-6e4f-444f-9836-fda5fac7d85b Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.131422932Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79172023-12-10T18:53:00.142Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.131477586Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79182023-12-10T18:53:00.142Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.131515732Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79192023-12-10T18:53:00.142Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.131601059Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79202023-12-10T18:53:00.142Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.131639127Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79212023-12-10T18:53:00.142Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.131756779Z","hostnameRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
79222023-12-10T18:53:00.142Z":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79232023-12-10T18:53:00.142Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.131865195Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79242023-12-10T18:53:00.142Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.131920291Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79252023-12-10T18:53:00.142Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.131959651Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79262023-12-10T18:53:00.142Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.132014633Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79272023-12-10T18:53:00.142Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.132053426Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79282023-12-10T18:53:00.143Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:00.132106743Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1418,"session_id":"e1856594-b734-4d14-81bb-0e652ea15c03"}
79292023-12-10T18:53:00.143ZLoading write count information from file "/tmp/test_repair_verify.data"
79302023-12-10T18:53:00.143ZRun Repair workload
79312023-12-10T18:53:00.143Z01/30 Write block 534 len 12288 data: 8 9 5
79322023-12-10T18:53:00.143Z02/30 Read block 503 len 36864
79332023-12-10T18:53:00.143Z03/30 Read block 251 len 32768
79342023-12-10T18:53:00.146Z04/30 Read block 580 len 40960
79352023-12-10T18:53:00.147Z05/30 Flush
79362023-12-10T18:53:00.147Z06/30 Flush
79372023-12-10T18:53:00.148Z07/30 Write block 121 len 20480 data: 9 9 11 13 11
79382023-12-10T18:53:00.148Z08/30 Write block 414 len 24576 data: 2 2 2 3 4 4
79392023-12-10T18:53:00.148Z09/30 Flush
79402023-12-10T18:53:00.162Z10/30 Write block 320 len 20480 data: 5 6 8 7 8
79412023-12-10T18:53:00.162Z11/30 Flush
79422023-12-10T18:53:00.175Z12/30 Write block 267 len 36864 data: 3 4 4 7 6 6 6 7 8
79432023-12-10T18:53:00.175Z13/30 Write block 448 len 32768 data: 4 4 4 4 4 5 4 4
79442023-12-10T18:53:00.175Z14/30 Read block 447 len 4096
79452023-12-10T18:53:00.175Z15/30 Write block 581 len 40960 data: 6 5 5 4 2 3 4 4 4 4
79462023-12-10T18:53:00.175Z16/30 Flush
79472023-12-10T18:53:00.175Z17/30 Write block 138 len 16384 data: 4 4 3 3
79482023-12-10T18:53:00.175Z18/30 Read block 579 len 24576
79492023-12-10T18:53:00.175Z19/30 Read block 330 len 28672
79502023-12-10T18:53:00.176Z20/30 Write block 26 len 28672 data: 5 3 3 4 4 5 7
79512023-12-10T18:53:00.176Z21/30 Read block 206 len 4096
79522023-12-10T18:53:00.183Z22/30 Read block 76 len 40960
79532023-12-10T18:53:00.183Z23/30 Read block 30 len 4096
79542023-12-10T18:53:00.183Z24/30 Read block 123 len 12288
79552023-12-10T18:53:00.183Z25/30 Read block 516 len 32768
79562023-12-10T18:53:00.184Z26/30 Flush
79572023-12-10T18:53:00.190Z27/30 Write block 57 len 4096 data: 4
79582023-12-10T18:53:00.190Z28/30 Write block 469 len 32768 data: 9 8 10 9 8 7 6 6
79592023-12-10T18:53:00.190Z29/30 Read block 30 len 8192
79602023-12-10T18:53:00.191Z30/30 Write block 265 len 20480 data: 5 4 4 5 5
79612023-12-10T18:53:00.191Z----------------------------------------------------------------
79622023-12-10T18:53:00.191Z Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30
79632023-12-10T18:53:00.191ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
79642023-12-10T18:53:00.191Z 1 Acked 1000 Write 3 Done Done Done false
79652023-12-10T18:53:00.191Z 2 Acked 1001 Read 9 Done Done Done false
79662023-12-10T18:53:00.192Z 3 Acked 1002 Read 8 Done Done Done false
79672023-12-10T18:53:00.192Z 4 Acked 1003 Read 10 Done Done Sent false
79682023-12-10T18:53:00.192Z 5 Acked 1004 Flush 0 Done Done Sent false
79692023-12-10T18:53:00.193Z 6 Acked 1005 Flush 0 Done Done Sent false
79702023-12-10T18:53:00.193Z 7 Acked 1006 Write 5 Done Done Sent false
79712023-12-10T18:53:00.193Z 8 Acked 1007 Write 6 Done Done Sent false
79722023-12-10T18:53:00.193Z 9 Acked 1008 Flush 0 Done Done Sent false
79732023-12-10T18:53:00.194Z 10 Acked 1009 Write 5 Done Done Sent false
79742023-12-10T18:53:00.194Z 11 Acked 1010 Flush 0 Done Done Sent false
79752023-12-10T18:53:00.194Z 12 Acked 1011 Write 9 Done Done Sent false
79762023-12-10T18:53:00.194Z 13 Acked 1012 Write 8 Done Done Sent false
79772023-12-10T18:53:00.194Z 14 Acked 1013 Read 1 Done Done Sent false
79782023-12-10T18:53:00.195Z 15 Acked 1014 Write 10 Done Done Sent false
79792023-12-10T18:53:00.195Z 16 Acked 1015 Flush 0 Done Done Sent false
79802023-12-10T18:53:00.195Z 17 Acked 1016 Write 4 Done Done Sent false
79812023-12-10T18:53:00.195Z 18 Acked 1017 Read 6 Done Done Sent false
79822023-12-10T18:53:00.195Z 19 Acked 1018 Read 7 Done Done Sent false
79832023-12-10T18:53:00.195Z 20 Acked 1019 Write 7 Done Done Sent false
79842023-12-10T18:53:00.196Z 21 Acked 1020 Read 1 Done Done Sent false
79852023-12-10T18:53:00.196Z 22 Acked 1021 Read 10 Done Done Sent false
79862023-12-10T18:53:00.196Z 23 Acked 1022 Read 1 Done Done Sent false
79872023-12-10T18:53:00.196Z 24 Acked 1023 Read 3 Done Done Sent false
79882023-12-10T18:53:00.196Z 25 Acked 1024 Read 8 Done Done Sent false
79892023-12-10T18:53:00.197Z 26 Acked 1025 Flush 0 Done Done Sent false
79902023-12-10T18:53:00.197Z 27 Acked 1026 Write 1 Done Done Sent false
79912023-12-10T18:53:00.197Z 28 Acked 1027 Write 8 Done Done Sent false
79922023-12-10T18:53:00.197Z 29 Acked 1028 Read 2 Done Done Sent false
79932023-12-10T18:53:00.197Z 30 Acked 1029 Write 5 Sent Sent Sent false
79942023-12-10T18:53:00.198Z STATES DS:0 DS:1 DS:2 TOTAL
79952023-12-10T18:53:00.198Z New 0 0 0 0
79962023-12-10T18:53:00.198Z Sent 1 1 27 29
79972023-12-10T18:53:00.198Z Done 29 29 3 61
79982023-12-10T18:53:00.198Z Skipped 0 0 0 0
79992023-12-10T18:53:00.199Z Error 0 0 0 0
80002023-12-10T18:53:00.199ZLast Flush: 1025 1025 0
80012023-12-10T18:53:00.199ZDownstairs last five completed:
80022023-12-10T18:53:00.199ZUpstairs last five completed: 30 29 28 27 26 25
80032023-12-10T18:53:00.199ZWrote out file "/tmp/test_repair_verify.data"
80042023-12-10T18:53:00.203Z
80052023-12-10T18:53:00.203ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
80062023-12-10T18:53:00.224ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
80072023-12-10T18:53:00.224Z 0 000-019 62 62 62 104 104 104 F F F
80082023-12-10T18:53:00.229Z 1 020-039 64 64 61 112 112 103 F F F <---
80092023-12-10T18:53:00.229Z 2 040-059 60 60 60 102 102 102 T T F <---
80102023-12-10T18:53:00.230Z 3 060-079 54 54 54 88 88 88 F F F
80112023-12-10T18:53:00.230Z 4 080-099 57 57 57 94 94 94 F F F
80122023-12-10T18:53:00.230Z 5 100-119 63 63 63 106 106 106 F F F
80132023-12-10T18:53:00.230Z 6 120-139 64 64 59 112 112 98 F F F <---
80142023-12-10T18:53:00.230Z 7 140-159 64 64 59 112 112 98 F F F <---
80152023-12-10T18:53:00.230Z 8 160-179 62 62 62 104 104 104 F F F
80162023-12-10T18:53:00.230Z 9 180-199 62 62 62 105 105 105 F F F
80172023-12-10T18:53:00.230Z 10 200-219 60 60 60 102 102 102 F F F
80182023-12-10T18:53:00.230Z 11 220-239 56 56 56 93 93 93 F F F
80192023-12-10T18:53:00.230Z 12 240-259 60 60 60 102 102 102 F F F
80202023-12-10T18:53:00.230Z 13 260-279 64 64 55 111 111 92 T T F <---
80212023-12-10T18:53:00.230Z 14 280-299 63 63 63 106 106 106 F F F
80222023-12-10T18:53:00.230Z 15 300-319 63 63 63 106 106 106 F F F
80232023-12-10T18:53:00.230Z 16 320-339 64 64 62 110 110 105 F F F <---
80242023-12-10T18:53:00.230Z 17 340-359 62 62 62 105 105 105 F F F
80252023-12-10T18:53:00.230Z 18 360-379 59 59 59 98 98 98 F F F
80262023-12-10T18:53:00.230Z 19 380-399 63 63 63 106 106 106 F F F
80272023-12-10T18:53:00.230Z 20 400-419 64 64 63 109 109 106 F F F <---
80282023-12-10T18:53:00.231Z 21 420-439 59 59 59 98 98 98 F F F
80292023-12-10T18:53:00.231Z 22 440-459 64 64 60 111 111 101 F F F <---
80302023-12-10T18:53:00.231Z 23 460-479 58 58 58 96 96 96 T T F <---
80312023-12-10T18:53:00.231Z 24 480-499 62 62 62 104 104 104 F F F
80322023-12-10T18:53:00.231Z 25 500-519 62 62 62 104 104 104 F F F
80332023-12-10T18:53:00.231Z 26 520-539 64 64 60 107 107 102 F F T <---
80342023-12-10T18:53:00.231Z 27 540-559 60 60 60 102 102 102 F F F
80352023-12-10T18:53:00.231Z 28 560-579 62 62 62 105 105 105 F F F
80362023-12-10T18:53:00.231Z 29 580-599 64 64 62 111 111 105 F F F <---
80372023-12-10T18:53:00.231ZMax gen: 64, Max flush: 112
80382023-12-10T18:53:00.231ZError: Difference in extent metadata found!
80392023-12-10T18:53:00.234ZA Difference in extent metadata is expected here
80402023-12-10T18:53:00.234ZOn loop 32
80412023-12-10T18:53:00.234Z
80422023-12-10T18:53:00.235ZVerifying data now
80432023-12-10T18:53:05.349ZNo change after verify
80442023-12-10T18:53:05.349ZLoop: 32 Downstairs dump after verify (and repair):
80452023-12-10T18:53:05.371ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
80462023-12-10T18:53:05.371Z 0 000-019 62 62 62 104 104 104 F F F
80472023-12-10T18:53:05.371Z 1 020-039 64 64 64 112 112 112 F F F
80482023-12-10T18:53:05.371Z 2 040-059 65 65 65 113 113 113 F F F
80492023-12-10T18:53:05.379Z 3 060-079 54 54 54 88 88 88 F F F
80502023-12-10T18:53:05.379Z 4 080-099 57 57 57 94 94 94 F F F
80512023-12-10T18:53:05.381Z 5 100-119 63 63 63 106 106 106 F F F
80522023-12-10T18:53:05.381Z 6 120-139 64 64 64 112 112 112 F F F
80532023-12-10T18:53:05.381Z 7 140-159 64 64 64 112 112 112 F F F
80542023-12-10T18:53:05.381Z 8 160-179 62 62 62 104 104 104 F F F
80552023-12-10T18:53:05.381Z 9 180-199 62 62 62 105 105 105 F F F
80562023-12-10T18:53:05.382Z 10 200-219 60 60 60 102 102 102 F F F
80572023-12-10T18:53:05.382Z 11 220-239 56 56 56 93 93 93 F F F
80582023-12-10T18:53:05.382Z 12 240-259 60 60 60 102 102 102 F F F
80592023-12-10T18:53:05.382Z 13 260-279 65 65 65 113 113 113 F F F
80602023-12-10T18:53:05.382Z 14 280-299 63 63 63 106 106 106 F F F
80612023-12-10T18:53:05.382Z 15 300-319 63 63 63 106 106 106 F F F
80622023-12-10T18:53:05.382Z 16 320-339 64 64 64 110 110 110 F F F
80632023-12-10T18:53:05.382Z 17 340-359 62 62 62 105 105 105 F F F
80642023-12-10T18:53:05.382Z 18 360-379 59 59 59 98 98 98 F F F
80652023-12-10T18:53:05.383Z 19 380-399 63 63 63 106 106 106 F F F
80662023-12-10T18:53:05.383Z 20 400-419 64 64 64 109 109 109 F F F
80672023-12-10T18:53:05.383Z 21 420-439 59 59 59 98 98 98 F F F
80682023-12-10T18:53:05.384Z 22 440-459 64 64 64 111 111 111 F F F
80692023-12-10T18:53:05.384Z 23 460-479 65 65 65 113 113 113 F F F
80702023-12-10T18:53:05.384Z 24 480-499 62 62 62 104 104 104 F F F
80712023-12-10T18:53:05.384Z 25 500-519 62 62 62 104 104 104 F F F
80722023-12-10T18:53:05.384Z 26 520-539 64 64 64 107 107 107 F F F
80732023-12-10T18:53:05.384Z 27 540-559 60 60 60 102 102 102 F F F
80742023-12-10T18:53:05.385Z 28 560-579 62 62 62 105 105 105 F F F
80752023-12-10T18:53:05.386Z 29 580-599 64 64 64 111 111 111 F F F
80762023-12-10T18:53:05.386ZMax gen: 65, Max flush: 113
80772023-12-10T18:53:05.387Z
80782023-12-10T18:53:05.387ZBegin loop 33
80792023-12-10T18:53:05.387ZDownstairs to restart: 2
80802023-12-10T18:53:05.408Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.406609408Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426}
80812023-12-10T18:53:05.408Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.407023119Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426}
80822023-12-10T18:53:05.408Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.407051559Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426}
80832023-12-10T18:53:05.408Z{"msg":"Crucible b8adadc8-f60a-4197-a8db-0a395060d776 has session id: c9b24003-c34c-4bef-8f0d-b11d53dcd1b8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.407102821Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
80842023-12-10T18:53:05.417Z{"msg":"Upstairs opts: Upstairs UUID: b8adadc8-f60a-4197-a8db-0a395060d776, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.407171658Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
80852023-12-10T18:53:05.417Z{"msg":"Crucible stats registered with UUID: b8adadc8-f60a-4197-a8db-0a395060d776","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.407233555Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
80862023-12-10T18:53:05.418ZCrucible runtime is spawned
80872023-12-10T18:53:05.418ZThe guest has requested activation with gen:66
80882023-12-10T18:53:05.419Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.40754121Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8","task":"up_listen"}
80892023-12-10T18:53:05.419Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.407577253Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"looper":"0","session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
80902023-12-10T18:53:05.419Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.407615159Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
80912023-12-10T18:53:05.420Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.407669769Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
80922023-12-10T18:53:05.420Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.407951925Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"looper":"1","session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
80932023-12-10T18:53:05.420Z{"msg":"[0] b8adadc8-f60a-4197-a8db-0a395060d776 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.408006691Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"looper":"0","session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
80942023-12-10T18:53:05.420Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.408146238Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"looper":"2","session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
80952023-12-10T18:53:05.420Z{"msg":"[1] b8adadc8-f60a-4197-a8db-0a395060d776 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.408228855Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"looper":"1","session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
80962023-12-10T18:53:05.420Z{"msg":"[2] b8adadc8-f60a-4197-a8db-0a395060d776 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.408318754Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"looper":"2","session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
80972023-12-10T18:53:05.420Z{"msg":"b8adadc8-f60a-4197-a8db-0a395060d776 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.408798917Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
80982023-12-10T18:53:05.420Z{"msg":"Set desired generation to :66","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.408883978Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
80992023-12-10T18:53:05.420Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.408983252Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81002023-12-10T18:53:05.420Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.409311325Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81012023-12-10T18:53:05.421Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.409392218Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81022023-12-10T18:53:05.421Z{"msg":"[0] b8adadc8-f60a-4197-a8db-0a395060d776 (c9b24003-c34c-4bef-8f0d-b11d53dcd1b8) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.409665376Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81032023-12-10T18:53:05.421Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.409811887Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81042023-12-10T18:53:05.421Z{"msg":"[0] client is_active_req TRUE, promote! session c9b24003-c34c-4bef-8f0d-b11d53dcd1b8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.409857117Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81052023-12-10T18:53:05.421Z{"msg":"[1] b8adadc8-f60a-4197-a8db-0a395060d776 (c9b24003-c34c-4bef-8f0d-b11d53dcd1b8) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.40989675Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81062023-12-10T18:53:05.421Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.409960618Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81072023-12-10T18:53:05.421Z{"msg":"[1] client is_active_req TRUE, promote! session c9b24003-c34c-4bef-8f0d-b11d53dcd1b8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.410005071Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81082023-12-10T18:53:05.421Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.410400093Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81092023-12-10T18:53:05.422Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.410491939Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81102023-12-10T18:53:05.422Z{"msg":"b8adadc8-f60a-4197-a8db-0a395060d776 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.410533879Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81112023-12-10T18:53:05.422Z{"msg":"[2] b8adadc8-f60a-4197-a8db-0a395060d776 (c9b24003-c34c-4bef-8f0d-b11d53dcd1b8) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.410631123Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81122023-12-10T18:53:05.422Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.410662327Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81132023-12-10T18:53:05.423Z{"msg":"[2] client is_active_req TRUE, promote! session c9b24003-c34c-4bef-8f0d-b11d53dcd1b8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.410701101Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81142023-12-10T18:53:05.423Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.410774084Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81152023-12-10T18:53:05.423Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.410823608Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81162023-12-10T18:53:05.423Z{"msg":"b8adadc8-f60a-4197-a8db-0a395060d776 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.410892895Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81172023-12-10T18:53:05.423Z{"msg":"[0] b8adadc8-f60a-4197-a8db-0a395060d776 (c9b24003-c34c-4bef-8f0d-b11d53dcd1b8) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.411186923Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81182023-12-10T18:53:05.423Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.411251917Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81192023-12-10T18:53:05.423Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:05.411276616Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81202023-12-10T18:53:05.423Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.411296983Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81212023-12-10T18:53:05.423Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.411317641Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81222023-12-10T18:53:05.423Z{"msg":"b8adadc8-f60a-4197-a8db-0a395060d776 WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.411344274Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81232023-12-10T18:53:05.424Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.411499614Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81242023-12-10T18:53:05.424Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.411564354Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81252023-12-10T18:53:05.424Z{"msg":"[1] b8adadc8-f60a-4197-a8db-0a395060d776 (c9b24003-c34c-4bef-8f0d-b11d53dcd1b8) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.411782427Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81262023-12-10T18:53:05.424Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.411840208Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81272023-12-10T18:53:05.424Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:05.411930455Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81282023-12-10T18:53:05.425Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.411997019Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81292023-12-10T18:53:05.425Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.412024855Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81302023-12-10T18:53:05.425Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.412045773Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81312023-12-10T18:53:05.425Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.41208599Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81322023-12-10T18:53:05.425Z{"msg":"b8adadc8-f60a-4197-a8db-0a395060d776 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.412109293Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81332023-12-10T18:53:05.425Z{"msg":"b8adadc8-f60a-4197-a8db-0a395060d776 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.41213219Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81342023-12-10T18:53:05.425Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.412172488Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81352023-12-10T18:53:05.426Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.412194324Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81362023-12-10T18:53:05.426Z{"msg":"[2] b8adadc8-f60a-4197-a8db-0a395060d776 (c9b24003-c34c-4bef-8f0d-b11d53dcd1b8) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.412636492Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81372023-12-10T18:53:05.426Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.412699239Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81382023-12-10T18:53:05.427Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:05.412724532Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81392023-12-10T18:53:05.427Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.412780375Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81402023-12-10T18:53:05.427Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.412830848Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81412023-12-10T18:53:05.427Z{"msg":"b8adadc8-f60a-4197-a8db-0a395060d776 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.41285431Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81422023-12-10T18:53:05.427ZThe guest has finished waiting for activation with:66
81432023-12-10T18:53:05.427Z{"msg":"[0]R flush_numbers[0..12]: [104, 112, 113, 88, 94, 106, 112, 112, 104, 105, 102, 93]","v":0,Wait for a query_work_queue command to finish before sending IO
81442023-12-10T18:53:05.427Z"name":"crucible","level":30,"time":"2023-12-10T18:53:05.412882384Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81452023-12-10T18:53:05.427Z{"msg":"[0]R generation[0..12]: [62, 64, 65, 54, 57, 63, 64, 64, 62, 62, 60, 56]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.41292495Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81462023-12-10T18:53:05.428Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.413031216Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81472023-12-10T18:53:05.428Z{"msg":"[1]R flush_numbers[0..12]: [104, 112, 113, 88, 94, 106, 112, 112, 104, 105, 102, 93]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.41306297Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81482023-12-10T18:53:05.428Z{"msg":"[1]R generation[0..12]: [62, 64, 65, 54, 57, 63, 64, 64, 62, 62, 60, 56]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.413084507Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81492023-12-10T18:53:05.428Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.413106169Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81502023-12-10T18:53:05.429Z{"msg":"[2]R flush_numbers[0..12]: [104, 112, 113, 88, 94, 106, 112, 112, 104, 105, 102, 93]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.41312841Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81512023-12-10T18:53:05.429Z{"msg":"[2]R generation[0..12]: [62, 64, 65, 54, 57, 63, 64, 64, 62, 62, 60, 56]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.413180481Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81522023-12-10T18:53:05.429Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.41320671Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81532023-12-10T18:53:05.429Z{"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.413254964Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81542023-12-10T18:53:05.429Z{"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.413278898Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81552023-12-10T18:53:05.429Z{"msg":"Next flush: 114","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.413336672Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81562023-12-10T18:53:05.429Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.413358616Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81572023-12-10T18:53:05.429Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.413378687Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81582023-12-10T18:53:05.429Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.413418643Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81592023-12-10T18:53:05.429Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.413440707Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81602023-12-10T18:53:05.430Z{"msg":"b8adadc8-f60a-4197-a8db-0a395060d776 is now active with session: c9b24003-c34c-4bef-8f0d-b11d53dcd1b8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.413462166Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81612023-12-10T18:53:05.430Z{"msg":"b8adadc8-f60a-4197-a8db-0a395060d776 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.413483365Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81622023-12-10T18:53:05.430Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.413503662Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81632023-12-10T18:53:05.430Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.413569073Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81642023-12-10T18:53:05.430Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.413594569Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81652023-12-10T18:53:05.430Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.41363619Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81662023-12-10T18:53:05.430Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.413658379Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81672023-12-10T18:53:05.431Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
81682023-12-10T18:53:05.431Z[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.413714659Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81692023-12-10T18:53:05.432Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.413760322Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81702023-12-10T18:53:05.432Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.413782732Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81712023-12-10T18:53:05.433Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.41382249Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81722023-12-10T18:53:05.433Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.413844688Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81732023-12-10T18:53:05.433Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:05.413884213Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1426,"session_id":"c9b24003-c34c-4bef-8f0d-b11d53dcd1b8"}
81742023-12-10T18:53:05.433ZLoading write count information from file "/tmp/test_repair_verify.data"
81752023-12-10T18:53:05.433ZRun Repair workload
81762023-12-10T18:53:05.440Z01/30 Write block 489 len 40960 data: 9 8 9 5 3 4 5 4 4 4
81772023-12-10T18:53:05.440Z02/30 Read block 174 len 28672
81782023-12-10T18:53:05.440Z03/30 Write block 182 len 20480 data: 6 7 5 5 4
81792023-12-10T18:53:05.440Z04/30 Read block 8 len 40960
81802023-12-10T18:53:05.440Z05/30 Read block 414 len 20480
81812023-12-10T18:53:05.440Z06/30 Write block 221 len 8192 data: 10 10
81822023-12-10T18:53:05.441Z07/30 Read block 514 len 28672
81832023-12-10T18:53:05.441Z08/30 Write block 350 len 20480 data: 5 6 7 8 6
81842023-12-10T18:53:05.441Z09/30 Read block 277 len 28672
81852023-12-10T18:53:05.441Z10/30 Write block 73 len 40960 data: 5 7 7 7 7 7 8 9 8 8
81862023-12-10T18:53:05.441Z11/30 Read block 138 len 16384
81872023-12-10T18:53:05.441Z12/30 Read block 540 len 4096
81882023-12-10T18:53:05.441Z13/30 Read block 121 len 16384
81892023-12-10T18:53:05.441Z14/30 Read block 551 len 40960
81902023-12-10T18:53:05.445Z15/30 Flush
81912023-12-10T18:53:05.445Z16/30 Write block 455 len 4096 data: 5
81922023-12-10T18:53:05.445Z17/30 Read block 445 len 20480
81932023-12-10T18:53:05.448Z18/30 Flush
81942023-12-10T18:53:05.452Z19/30 Write block 116 len 8192 data: 6 7
81952023-12-10T18:53:05.452Z20/30 Write block 226 len 8192 data: 7 6
81962023-12-10T18:53:05.452Z21/30 Read block 459 len 24576
81972023-12-10T18:53:05.457Z22/30 Read block 556 len 24576
81982023-12-10T18:53:05.457Z23/30 Read block 525 len 16384
81992023-12-10T18:53:05.460Z24/30 Write block 7 len 40960 data: 5 5 5 5 5 6 8 8 7 7
82002023-12-10T18:53:05.460Z25/30 Write block 256 len 4096 data: 5
82012023-12-10T18:53:05.460Z26/30 Write block 565 len 12288 data: 7 7 6
82022023-12-10T18:53:05.464Z27/30 Write block 13 len 36864 data: 9 9 8 8 9 7 7 5 4
82032023-12-10T18:53:05.464Z28/30 Write block 79 len 4096 data: 9
82042023-12-10T18:53:05.464Z29/30 Write block 24 len 20480 data: 5 5 6 4 4
82052023-12-10T18:53:05.464Z30/30 Read block 399 len 28672
82062023-12-10T18:53:05.487Z----------------------------------------------------------------
82072023-12-10T18:53:05.488Z Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30
82082023-12-10T18:53:05.488ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
82092023-12-10T18:53:05.488Z 1 Acked 1000 Write 10 Done Done Done false
82102023-12-10T18:53:05.488Z 2 Acked 1001 Read 7 Done Done Sent false
82112023-12-10T18:53:05.488Z 3 Acked 1002 Write 5 Done Done Sent false
82122023-12-10T18:53:05.489Z 4 Acked 1003 Read 10 Done Done Sent false
82132023-12-10T18:53:05.489Z 5 Acked 1004 Read 5 Done Done Sent false
82142023-12-10T18:53:05.489Z 6 Acked 1005 Write 2 Done Done Sent false
82152023-12-10T18:53:05.489Z 7 Acked 1006 Read 7 Done Done Sent false
82162023-12-10T18:53:05.489Z 8 Acked 1007 Write 5 Done Done Sent false
82172023-12-10T18:53:05.491Z 9 Acked 1008 Read 7 Done Done Sent false
82182023-12-10T18:53:05.491Z 10 Acked 1009 Write 10 Done Done Sent false
82192023-12-10T18:53:05.491Z 11 Acked 1010 Read 4 Done Done Sent false
82202023-12-10T18:53:05.492Z 12 Acked 1011 Read 1 Done Done Sent false
82212023-12-10T18:53:05.492Z 13 Acked 1012 Read 4 Done Done Sent false
82222023-12-10T18:53:05.492Z 14 Acked 1013 Read 10 Done Done Sent false
82232023-12-10T18:53:05.493Z 15 Acked 1014 Flush 0 Done Done Sent false
82242023-12-10T18:53:05.493Z 16 Acked 1015 Write 1 Done Done Sent false
82252023-12-10T18:53:05.494Z 17 Acked 1016 Read 5 Done Done Sent false
82262023-12-10T18:53:05.494Z 18 Acked 1017 Flush 0 Done Done Sent false
82272023-12-10T18:53:05.494Z 19 Acked 1018 Write 2 Done Done Sent false
82282023-12-10T18:53:05.494Z 20 Acked 1019 Write 2 Done Done Sent false
82292023-12-10T18:53:05.495Z 21 Acked 1020 Read 6 Done Done Sent false
82302023-12-10T18:53:05.495Z 22 Acked 1021 Read 6 Done Done Sent false
82312023-12-10T18:53:05.495Z 23 Acked 1022 Read 4 Done Done Sent false
82322023-12-10T18:53:05.495Z 24 Acked 1023 Write 10 Done Done Sent false
82332023-12-10T18:53:05.495Z 25 Acked 1024 Write 1 Done Done Sent false
82342023-12-10T18:53:05.495Z 26 Acked 1025 Write 3 Done Done Sent false
82352023-12-10T18:53:05.496Z 27 Acked 1026 Write 9 Done Done Sent false
82362023-12-10T18:53:05.496Z 28 Acked 1027 Write 1 Done Done Sent false
82372023-12-10T18:53:05.496Z 29 Acked 1028 Write 5 Done Done Sent false
82382023-12-10T18:53:05.496Z 30 Acked 1029 Read 7 Done Sent Sent false
82392023-12-10T18:53:05.497Z STATES DS:0 DS:1 DS:2 TOTAL
82402023-12-10T18:53:05.497Z New 0 0 0 0
82412023-12-10T18:53:05.497Z Sent 0 1 29 30
82422023-12-10T18:53:05.498Z Done 30 29 1 60
82432023-12-10T18:53:05.498Z Skipped 0 0 0 0
82442023-12-10T18:53:05.498Z Error 0 0 0 0
82452023-12-10T18:53:05.499ZLast Flush: 1017 1017 0
82462023-12-10T18:53:05.499ZDownstairs last five completed:
82472023-12-10T18:53:05.499ZUpstairs last five completed: 30 29 28 27 26 25
82482023-12-10T18:53:05.499ZWrote out file "/tmp/test_repair_verify.data"
82492023-12-10T18:53:05.937Z
82502023-12-10T18:53:05.937ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
82512023-12-10T18:53:05.937ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
82522023-12-10T18:53:05.937Z 0 000-019 62 62 62 104 104 104 T T F <---
82532023-12-10T18:53:05.937Z 1 020-039 64 64 64 112 112 112 T T F <---
82542023-12-10T18:53:05.937Z 2 040-059 65 65 65 113 113 113 F F F
82552023-12-10T18:53:05.937Z 3 060-079 66 66 54 114 114 88 T T F <---
82562023-12-10T18:53:05.939Z 4 080-099 66 66 57 114 114 94 F F F <---
82572023-12-10T18:53:05.939Z 5 100-119 63 63 63 106 106 106 T T F <---
82582023-12-10T18:53:05.939Z 6 120-139 64 64 64 112 112 112 F F F
82592023-12-10T18:53:05.939Z 7 140-159 64 64 64 112 112 112 F F F
82602023-12-10T18:53:05.939Z 8 160-179 62 62 62 104 104 104 F F F
82612023-12-10T18:53:05.939Z 9 180-199 66 66 62 114 114 105 F F F <---
82622023-12-10T18:53:05.939Z 10 200-219 60 60 60 102 102 102 F F F
82632023-12-10T18:53:05.939Z 11 220-239 66 66 56 114 114 93 T T F <---
82642023-12-10T18:53:05.939Z 12 240-259 60 60 60 102 102 102 T T F <---
82652023-12-10T18:53:05.939Z 13 260-279 65 65 65 113 113 113 F F F
82662023-12-10T18:53:05.941Z 14 280-299 63 63 63 106 106 106 F F F
82672023-12-10T18:53:05.941Z 15 300-319 63 63 63 106 106 106 F F F
82682023-12-10T18:53:05.941Z 16 320-339 64 64 64 110 110 110 F F F
82692023-12-10T18:53:05.941Z 17 340-359 66 66 62 114 114 105 F F F <---
82702023-12-10T18:53:05.941Z 18 360-379 59 59 59 98 98 98 F F F
82712023-12-10T18:53:05.941Z 19 380-399 63 63 63 106 106 106 F F F
82722023-12-10T18:53:05.941Z 20 400-419 64 64 64 109 109 109 F F F
82732023-12-10T18:53:05.941Z 21 420-439 59 59 59 98 98 98 F F F
82742023-12-10T18:53:05.941Z 22 440-459 66 66 64 115 115 111 F F F <---
82752023-12-10T18:53:05.941Z 23 460-479 65 65 65 113 113 113 F F F
82762023-12-10T18:53:05.941Z 24 480-499 66 66 62 114 114 104 F F T <---
82772023-12-10T18:53:05.941Z 25 500-519 62 62 62 104 104 104 F F F
82782023-12-10T18:53:05.941Z 26 520-539 64 64 64 107 107 107 F F F
82792023-12-10T18:53:05.941Z 27 540-559 60 60 60 102 102 102 F F F
82802023-12-10T18:53:05.942Z 28 560-579 62 62 62 105 105 105 T T F <---
82812023-12-10T18:53:05.942Z 29 580-599 64 64 64 111 111 111 F F F
82822023-12-10T18:53:05.942ZMax gen: 66, Max flush: 115
82832023-12-10T18:53:05.942ZError: Difference in extent metadata found!
82842023-12-10T18:53:05.942ZA Difference in extent metadata is expected here
82852023-12-10T18:53:05.942ZOn loop 33
82862023-12-10T18:53:05.942Z
82872023-12-10T18:53:05.942ZVerifying data now
82882023-12-10T18:53:10.716ZNo change after verify
82892023-12-10T18:53:10.717ZLoop: 33 Downstairs dump after verify (and repair):
82902023-12-10T18:53:10.744ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
82912023-12-10T18:53:10.745Z 0 000-019 67 67 67 116 116 116 F F F
82922023-12-10T18:53:10.745Z 1 020-039 67 67 67 116 116 116 F F F
82932023-12-10T18:53:10.745Z 2 040-059 65 65 65 113 113 113 F F F
82942023-12-10T18:53:10.745Z 3 060-079 67 67 67 116 116 116 F F F
82952023-12-10T18:53:10.745Z 4 080-099 66 66 66 114 114 114 F F F
82962023-12-10T18:53:10.745Z 5 100-119 67 67 67 116 116 116 F F F
82972023-12-10T18:53:10.745Z 6 120-139 64 64 64 112 112 112 F F F
82982023-12-10T18:53:10.745Z 7 140-159 64 64 64 112 112 112 F F F
82992023-12-10T18:53:10.749Z 8 160-179 62 62 62 104 104 104 F F F
83002023-12-10T18:53:10.750Z 9 180-199 66 66 66 114 114 114 F F F
83012023-12-10T18:53:10.750Z 10 200-219 60 60 60 102 102 102 F F F
83022023-12-10T18:53:10.751Z 11 220-239 67 67 67 116 116 116 F F F
83032023-12-10T18:53:10.751Z 12 240-259 67 67 67 116 116 116 F F F
83042023-12-10T18:53:10.751Z 13 260-279 65 65 65 113 113 113 F F F
83052023-12-10T18:53:10.751Z 14 280-299 63 63 63 106 106 106 F F F
83062023-12-10T18:53:10.752Z 15 300-319 63 63 63 106 106 106 F F F
83072023-12-10T18:53:10.752Z 16 320-339 64 64 64 110 110 110 F F F
83082023-12-10T18:53:10.753Z 17 340-359 66 66 66 114 114 114 F F F
83092023-12-10T18:53:10.753Z 18 360-379 59 59 59 98 98 98 F F F
83102023-12-10T18:53:10.753Z 19 380-399 63 63 63 106 106 106 F F F
83112023-12-10T18:53:10.753Z 20 400-419 64 64 64 109 109 109 F F F
83122023-12-10T18:53:10.753Z 21 420-439 59 59 59 98 98 98 F F F
83132023-12-10T18:53:10.754Z 22 440-459 66 66 66 115 115 115 F F F
83142023-12-10T18:53:10.754Z 23 460-479 65 65 65 113 113 113 F F F
83152023-12-10T18:53:10.754Z 24 480-499 66 66 66 114 114 114 F F F
83162023-12-10T18:53:10.754Z 25 500-519 62 62 62 104 104 104 F F F
83172023-12-10T18:53:10.755Z 26 520-539 64 64 64 107 107 107 F F F
83182023-12-10T18:53:10.755Z 27 540-559 60 60 60 102 102 102 F F F
83192023-12-10T18:53:10.755Z 28 560-579 67 67 67 116 116 116 F F F
83202023-12-10T18:53:10.756Z 29 580-599 64 64 64 111 111 111 F F F
83212023-12-10T18:53:10.756ZMax gen: 67, Max flush: 116
83222023-12-10T18:53:10.756Z
83232023-12-10T18:53:10.756ZBegin loop 34
83242023-12-10T18:53:10.757ZDownstairs to restart: 2
83252023-12-10T18:53:10.781Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.777218725Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434}
83262023-12-10T18:53:10.783Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.777651067Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434}
83272023-12-10T18:53:10.783Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.777704987Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434}
83282023-12-10T18:53:10.783Z{"msg":"Crucible 2a06291a-28a8-4b5e-a6b3-f29214165bb4 has session id: 6581aa42-f693-48a6-bb55-d3105ea83d3e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.777729448Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83292023-12-10T18:53:10.783Z{"msg":"Upstairs opts: Upstairs UUID: 2a06291a-28a8-4b5e-a6b3-f29214165bb4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.777787683Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83302023-12-10T18:53:10.783Z{"msg":"Crucible stats registered with UUID: 2a06291a-28a8-4b5e-a6b3-f29214165bb4","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.777814228Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83312023-12-10T18:53:10.783ZCrucible runtime is spawned
83322023-12-10T18:53:10.783ZThe guest has requested activation with gen:68
83332023-12-10T18:53:10.784Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.778066555Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e","task":"up_listen"}
83342023-12-10T18:53:10.784Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.778131105Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"looper":"2","session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83352023-12-10T18:53:10.784Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.778171793Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83362023-12-10T18:53:10.784Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.778240233Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83372023-12-10T18:53:10.785Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.778388628Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"looper":"0","session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83382023-12-10T18:53:10.785Z{"msg":"[2] 2a06291a-28a8-4b5e-a6b3-f29214165bb4 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.778460614Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"looper":"2","session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83392023-12-10T18:53:10.785Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.778516558Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"looper":"1","session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83402023-12-10T18:53:10.785Z{"msg":"[0] 2a06291a-28a8-4b5e-a6b3-f29214165bb4 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.778647188Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"looper":"0","session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83412023-12-10T18:53:10.785Z{"msg":"[1] 2a06291a-28a8-4b5e-a6b3-f29214165bb4 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.77874664Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"looper":"1","session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83422023-12-10T18:53:10.785Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.779120012Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83432023-12-10T18:53:10.785Z{"msg":"2a06291a-28a8-4b5e-a6b3-f29214165bb4 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.779179168Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83442023-12-10T18:53:10.785Z{"msg":"Set desired generation to :68","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.779203775Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83452023-12-10T18:53:10.785Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.779530962Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83462023-12-10T18:53:10.786Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.779597049Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83472023-12-10T18:53:10.787Z{"msg":"[1] 2a06291a-28a8-4b5e-a6b3-f29214165bb4 (6581aa42-f693-48a6-bb55-d3105ea83d3e) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.780172803Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83482023-12-10T18:53:10.787Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.780315822Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83492023-12-10T18:53:10.787Z{"msg":"[1] client is_active_req TRUE, promote! session 6581aa42-f693-48a6-bb55-d3105ea83d3e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.780389129Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83502023-12-10T18:53:10.787Z{"msg":"[0] 2a06291a-28a8-4b5e-a6b3-f29214165bb4 (6581aa42-f693-48a6-bb55-d3105ea83d3e) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.780419453Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83512023-12-10T18:53:10.787Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.780454951Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83522023-12-10T18:53:10.787Z{"msg":"[0] client is_active_req TRUE, promote! session 6581aa42-f693-48a6-bb55-d3105ea83d3e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.780492857Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83532023-12-10T18:53:10.787Z{"msg":"[2] 2a06291a-28a8-4b5e-a6b3-f29214165bb4 (6581aa42-f693-48a6-bb55-d3105ea83d3e) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.780833142Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83542023-12-10T18:53:10.787Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.780958766Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83552023-12-10T18:53:10.787Z{"msg":"[2] client is_active_req TRUE, promote! session 6581aa42-f693-48a6-bb55-d3105ea83d3e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.781001802Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83562023-12-10T18:53:10.787Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.781041004Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83572023-12-10T18:53:10.788Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.781091934Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83582023-12-10T18:53:10.788Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.781115446Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83592023-12-10T18:53:10.788Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.781143757Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83602023-12-10T18:53:10.788Z{"msg":"2a06291a-28a8-4b5e-a6b3-f29214165bb4 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.781201684Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83612023-12-10T18:53:10.788Z{"msg":"2a06291a-28a8-4b5e-a6b3-f29214165bb4 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.781230476Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83622023-12-10T18:53:10.797Z{"msg":"[0] 2a06291a-28a8-4b5e-a6b3-f29214165bb4 (6581aa42-f693-48a6-bb55-d3105ea83d3e) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.781827543Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83632023-12-10T18:53:10.797Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.781908692Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83642023-12-10T18:53:10.798Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:10.781952017Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83652023-12-10T18:53:10.798Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.782023652Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83662023-12-10T18:53:10.798Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.782066814Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83672023-12-10T18:53:10.798Z{"msg":"2a06291a-28a8-4b5e-a6b3-f29214165bb4 WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.782183576Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83682023-12-10T18:53:10.798Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.782226371Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83692023-12-10T18:53:10.798Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.782267409Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83702023-12-10T18:53:10.798Z{"msg":"[1] 2a06291a-28a8-4b5e-a6b3-f29214165bb4 (6581aa42-f693-48a6-bb55-d3105ea83d3e) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.782311733Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83712023-12-10T18:53:10.798Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.782389454Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83722023-12-10T18:53:10.798Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:10.782430533Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83732023-12-10T18:53:10.798Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.782492303Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83742023-12-10T18:53:10.798Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.782516565Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83752023-12-10T18:53:10.798Z{"msg":"2a06291a-28a8-4b5e-a6b3-f29214165bb4 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.782537657Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83762023-12-10T18:53:10.798Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.782592288Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83772023-12-10T18:53:10.798Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.782616618Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83782023-12-10T18:53:10.799Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.782638056Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83792023-12-10T18:53:10.799Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.782683979Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83802023-12-10T18:53:10.799Z{"msg":"2a06291a-28a8-4b5e-a6b3-f29214165bb4 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.782707723Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83812023-12-10T18:53:10.799Z{"msg":"[2] 2a06291a-28a8-4b5e-a6b3-f29214165bb4 (6581aa42-f693-48a6-bb55-d3105ea83d3e) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.783103726Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83822023-12-10T18:53:10.799Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.783182164Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83832023-12-10T18:53:10.799Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:10.783207352Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83842023-12-10T18:53:10.799Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.783234219Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83852023-12-10T18:53:10.799Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.783291488Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83862023-12-10T18:53:10.799Z{"msg":"2a06291a-28a8-4b5e-a6b3-f29214165bb4 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.783318284Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83872023-12-10T18:53:10.800Z{"msg":"[0]R flush_numbers[0..12]: [116, 116, 113, 116, 114, 116, 112, 112, 104, 114, 102, 116]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.783377853Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83882023-12-10T18:53:10.800Z{"msg":"[0]R generation[0..12]: [67, 67, 65, 67, 66, 67, 64, 64, 62, 66, 60, 67]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.78340434Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83892023-12-10T18:53:10.800Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"nameThe guest has finished waiting for activation with:68
83902023-12-10T18:53:10.800Z":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
83912023-12-10T18:53:10.800Z,"time":"2023-12-10T18:53:10.783490263Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83922023-12-10T18:53:10.800Z{"msg":"[1]R flush_numbers[0..12]: [116, 116, 113, 116, 114, 116, 112, 112, 104, 114, 102, 116]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.783740363Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83932023-12-10T18:53:10.800Z{"msg":"[1]R generation[0..12]: [67, 67, 65, 67, 66, 67, 64, 64, 62, 66, 60, 67]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.783780112Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83942023-12-10T18:53:10.800Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.783816275Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83952023-12-10T18:53:10.800Z{"msg":"[2]R flush_numbers[0..12]: [116, 116, 113, 116, 114, 116, 112, 112, 104, 114, 102, 116]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.783856334Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83962023-12-10T18:53:10.800Z{"msg":"[2]R generation[0..12]: [67, 67, 65, 67, 66, 67, 64, 64, 62, 66, 60, 67]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.783966652Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83972023-12-10T18:53:10.800Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.784009165Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83982023-12-10T18:53:10.801Z{"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.784045633Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
83992023-12-10T18:53:10.801Z{"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.784107457Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
84002023-12-10T18:53:10.801Z{"msg":"Next flush: 117","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.784145622Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
84012023-12-10T18:53:10.801Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.784201704Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
84022023-12-10T18:53:10.801Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.784240303Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
84032023-12-10T18:53:10.801Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.784301516Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
84042023-12-10T18:53:10.802Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.784339776Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
84052023-12-10T18:53:10.802Z{"msg":"2a06291a-28a8-4b5e-a6b3-f29214165bb4 is now active with session: 6581aa42-f693-48a6-bb55-d3105ea83d3e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.784398408Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
84062023-12-10T18:53:10.802Z,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
84072023-12-10T18:53:10.802Z{"msg":"2a06291a-28a8-4b5e-a6b3-f29214165bb4 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.784449054Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
84082023-12-10T18:53:10.802Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.784501015Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
84092023-12-10T18:53:10.802Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.784539073Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
84102023-12-10T18:53:10.802Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.784588823Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
84112023-12-10T18:53:10.802Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.784611703Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
84122023-12-10T18:53:10.802Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.784631381Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
84132023-12-10T18:53:10.802Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.784678658Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
84142023-12-10T18:53:10.802Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.784728367Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
84152023-12-10T18:53:10.802Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.784751434Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
84162023-12-10T18:53:10.802Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.784792335Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
84172023-12-10T18:53:10.802Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.784814463Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
84182023-12-10T18:53:10.802Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:10.784853761Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1434,"session_id":"6581aa42-f693-48a6-bb55-d3105ea83d3e"}
84192023-12-10T18:53:10.802ZLoading write count information from file "/tmp/test_repair_verify.data"
84202023-12-10T18:53:10.802ZRun Repair workload
84212023-12-10T18:53:10.803Z01/30 Write block 373 len 32768 data: 4 4 6 6 6 9 10 11
84222023-12-10T18:53:10.803Z02/30 Write block 81 len 32768 data: 9 9 9 8 10 11 9 8
84232023-12-10T18:53:10.803Z03/30 Read block 40 len 24576
84242023-12-10T18:53:10.803Z04/30 Read block 550 len 36864
84252023-12-10T18:53:10.803Z05/30 Read block 301 len 32768
84262023-12-10T18:53:10.808Z06/30 Write block 397 len 40960 data: 4 5 6 6 6 6 7 7 7 6
84272023-12-10T18:53:10.808Z07/30 Read block 224 len 20480
84282023-12-10T18:53:10.808Z08/30 Write block 353 len 28672 data: 9 7 5 6 6 5 5
84292023-12-10T18:53:10.808Z09/30 Read block 153 len 16384
84302023-12-10T18:53:10.808Z10/30 Write block 505 len 36864 data: 5 7 7 7 6 7 7 6 7
84312023-12-10T18:53:10.809Z11/30 Write block 203 len 8192 data: 7 6
84322023-12-10T18:53:10.809Z12/30 Read block 198 len 4096
84332023-12-10T18:53:10.809Z13/30 Write block 100 len 16384 data: 5 4 6 6
84342023-12-10T18:53:10.809Z14/30 Read block 39 len 24576
84352023-12-10T18:53:10.813Z15/30 Write block 196 len 24576 data: 7 8 7 7 6 6
84362023-12-10T18:53:10.813Z16/30 Flush
84372023-12-10T18:53:10.816Z17/30 Write block 402 len 32768 data: 7 8 8 8 7 7 7 5
84382023-12-10T18:53:10.816Z18/30 Flush
84392023-12-10T18:53:10.819Z19/30 Write block 74 len 24576 data: 8 8 8 8 8 10
84402023-12-10T18:53:10.820Z20/30 Flush
84412023-12-10T18:53:10.824Z21/30 Write block 562 len 32768 data: 6 6 7 8 8 7 6 7
84422023-12-10T18:53:10.830Z22/30 Read block 329 len 28672
84432023-12-10T18:53:10.831Z23/30 Read block 256 len 20480
84442023-12-10T18:53:10.837Z24/30 Write block 548 len 32768 data: 7 4 3 3 3 3 3 4
84452023-12-10T18:53:10.837Z25/30 Read block 548 len 20480
84462023-12-10T18:53:10.837Z26/30 Flush
84472023-12-10T18:53:10.838Z27/30 Write block 246 len 4096 data: 9
84482023-12-10T18:53:10.838Z28/30 Read block 154 len 28672
84492023-12-10T18:53:10.844Z29/30 Write block 252 len 20480 data: 8 9 6 5 6
84502023-12-10T18:53:10.844Z30/30 Read block 546 len 12288
84512023-12-10T18:53:10.845Z----------------------------------------------------------------
84522023-12-10T18:53:10.845Z Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30
84532023-12-10T18:53:10.845ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
84542023-12-10T18:53:10.845Z 1 Acked 1000 Write 8 Done Done Done false
84552023-12-10T18:53:10.845Z 2 Acked 1001 Write 8 Done Done Done false
84562023-12-10T18:53:10.845Z 3 Acked 1002 Read 6 Done Done Done false
84572023-12-10T18:53:10.845Z 4 Acked 1003 Read 9 Done Done Sent false
84582023-12-10T18:53:10.845Z 5 Acked 1004 Read 8 Done Done Sent false
84592023-12-10T18:53:10.845Z 6 Acked 1005 Write 10 Done Done Sent false
84602023-12-10T18:53:10.846Z 7 Acked 1006 Read 5 Done Done Sent false
84612023-12-10T18:53:10.846Z 8 Acked 1007 Write 7 Done Done Sent false
84622023-12-10T18:53:10.846Z 9 Acked 1008 Read 4 Done Done Sent false
84632023-12-10T18:53:10.846Z 10 Acked 1009 Write 9 Done Done Sent false
84642023-12-10T18:53:10.846Z 11 Acked 1010 Write 2 Done Done Sent false
84652023-12-10T18:53:10.846Z 12 Acked 1011 Read 1 Done Done Sent false
84662023-12-10T18:53:10.846Z 13 Acked 1012 Write 4 Done Done Sent false
84672023-12-10T18:53:10.846Z 14 Acked 1013 Read 6 Done Done Sent false
84682023-12-10T18:53:10.846Z 15 Acked 1014 Write 6 Done Done Sent false
84692023-12-10T18:53:10.847Z 16 Acked 1015 Flush 0 Done Done Sent false
84702023-12-10T18:53:10.847Z 17 Acked 1016 Write 8 Done Done Sent false
84712023-12-10T18:53:10.847Z 18 Acked 1017 Flush 0 Done Done Sent false
84722023-12-10T18:53:10.847Z 19 Acked 1018 Write 6 Done Done Sent false
84732023-12-10T18:53:10.847Z 20 Acked 1019 Flush 0 Done Done Sent false
84742023-12-10T18:53:10.847Z 21 Acked 1020 Write 8 Done Done Sent false
84752023-12-10T18:53:10.847Z 22 Acked 1021 Read 7 Done Done Sent false
84762023-12-10T18:53:10.847Z 23 Acked 1022 Read 5 Done Done Sent false
84772023-12-10T18:53:10.847Z 24 Acked 1023 Write 8 Done Done Sent false
84782023-12-10T18:53:10.848Z 25 Acked 1024 Read 5 Done Done Sent false
84792023-12-10T18:53:10.848Z 26 Acked 1025 Flush 0 Done Done Sent false
84802023-12-10T18:53:10.848Z 27 Acked 1026 Write 1 Done Done Sent false
84812023-12-10T18:53:10.848Z 28 Acked 1027 Read 7 Done Done Sent false
84822023-12-10T18:53:10.848Z 29 Acked 1028 Write 5 Done Done Sent false
84832023-12-10T18:53:10.848Z 30 Acked 1029 Read 3 Sent Done Sent false
84842023-12-10T18:53:10.848Z STATES DS:0 DS:1 DS:2 TOTAL
84852023-12-10T18:53:10.848Z New 0 0 0 0
84862023-12-10T18:53:10.848Z Sent 1 0 27 28
84872023-12-10T18:53:10.848Z Done 29 30 3 62
84882023-12-10T18:53:10.848Z Skipped 0 0 0 0
84892023-12-10T18:53:10.848Z Error 0 0 0 0
84902023-12-10T18:53:10.848ZLast Flush: 1025 1025 0
84912023-12-10T18:53:10.848ZDownstairs last five completed:
84922023-12-10T18:53:10.848ZUpstairs last five completed: 30 29 28 27 26 25
84932023-12-10T18:53:10.848ZWrote out file "/tmp/test_repair_verify.data"
84942023-12-10T18:53:10.856Z
84952023-12-10T18:53:10.856ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
84962023-12-10T18:53:10.889ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
84972023-12-10T18:53:10.889Z 0 000-019 67 67 67 116 116 116 F F F
84982023-12-10T18:53:10.889Z 1 020-039 67 67 67 116 116 116 F F F
84992023-12-10T18:53:10.889Z 2 040-059 65 65 65 113 113 113 F F F
85002023-12-10T18:53:10.889Z 3 060-079 68 68 67 119 119 116 F F F <---
85012023-12-10T18:53:10.889Z 4 080-099 68 68 66 117 117 114 F F T <---
85022023-12-10T18:53:10.889Z 5 100-119 68 68 67 117 117 116 F F F <---
85032023-12-10T18:53:10.889Z 6 120-139 64 64 64 112 112 112 F F F
85042023-12-10T18:53:10.890Z 7 140-159 64 64 64 112 112 112 F F F
85052023-12-10T18:53:10.890Z 8 160-179 62 62 62 104 104 104 F F F
85062023-12-10T18:53:11.308Z 9 180-199 68 68 66 117 117 114 F F F <---
85072023-12-10T18:53:11.308Z 10 200-219 68 68 60 117 117 102 F F F <---
85082023-12-10T18:53:11.308Z 11 220-239 67 67 67 116 116 116 F F F
85092023-12-10T18:53:11.308Z 12 240-259 67 67 67 116 116 116 T T F <---
85102023-12-10T18:53:11.308Z 13 260-279 65 65 65 113 113 113 F F F
85112023-12-10T18:53:11.308Z 14 280-299 63 63 63 106 106 106 F F F
85122023-12-10T18:53:11.308Z 15 300-319 63 63 63 106 106 106 F F F
85132023-12-10T18:53:11.308Z 16 320-339 64 64 64 110 110 110 F F F
85142023-12-10T18:53:11.308Z 17 340-359 68 68 66 117 117 114 F F F <---
85152023-12-10T18:53:11.308Z 18 360-379 68 68 59 117 117 98 F F T <---
85162023-12-10T18:53:11.308Z 19 380-399 68 68 63 117 117 106 F F T <---
85172023-12-10T18:53:11.309Z 20 400-419 68 68 64 118 118 109 F F F <---
85182023-12-10T18:53:11.309Z 21 420-439 59 59 59 98 98 98 F F F
85192023-12-10T18:53:11.309Z 22 440-459 66 66 66 115 115 115 F F F
85202023-12-10T18:53:11.309Z 23 460-479 65 65 65 113 113 113 F F F
85212023-12-10T18:53:11.309Z 24 480-499 66 66 66 114 114 114 F F F
85222023-12-10T18:53:11.309Z 25 500-519 68 68 62 117 117 104 F F F <---
85232023-12-10T18:53:11.309Z 26 520-539 64 64 64 107 107 107 F F F
85242023-12-10T18:53:11.309Z 27 540-559 68 68 60 120 120 102 F F F <---
85252023-12-10T18:53:11.309Z 28 560-579 68 68 67 120 120 116 F F F <---
85262023-12-10T18:53:11.309Z 29 580-599 64 64 64 111 111 111 F F F
85272023-12-10T18:53:11.309ZMax gen: 68, Max flush: 120
85282023-12-10T18:53:11.309ZError: Difference in extent metadata found!
85292023-12-10T18:53:11.309ZA Difference in extent metadata is expected here
85302023-12-10T18:53:11.309ZOn loop 34
85312023-12-10T18:53:11.309Z
85322023-12-10T18:53:11.309ZVerifying data now
85332023-12-10T18:53:17.407ZNo change after verify
85342023-12-10T18:53:17.408ZLoop: 34 Downstairs dump after verify (and repair):
85352023-12-10T18:53:17.507ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85362023-12-10T18:53:17.507Z 0 000-019 67 67 67 116 116 116 F F F
85372023-12-10T18:53:17.507Z 1 020-039 67 67 67 116 116 116 F F F
85382023-12-10T18:53:17.507Z 2 040-059 65 65 65 113 113 113 F F F
85392023-12-10T18:53:17.508Z 3 060-079 68 68 68 119 119 119 F F F
85402023-12-10T18:53:17.508Z 4 080-099 68 68 68 117 117 117 F F F
85412023-12-10T18:53:17.508Z 5 100-119 68 68 68 117 117 117 F F F
85422023-12-10T18:53:17.508Z 6 120-139 64 64 64 112 112 112 F F F
85432023-12-10T18:53:17.508Z 7 140-159 64 64 64 112 112 112 F F F
85442023-12-10T18:53:17.508Z 8 160-179 62 62 62 104 104 104 F F F
85452023-12-10T18:53:17.509Z 9 180-199 68 68 68 117 117 117 F F F
85462023-12-10T18:53:17.509Z 10 200-219 68 68 68 117 117 117 F F F
85472023-12-10T18:53:17.509Z 11 220-239 67 67 67 116 116 116 F F F
85482023-12-10T18:53:17.509Z 12 240-259 69 69 69 121 121 121 F F F
85492023-12-10T18:53:17.509Z 13 260-279 65 65 65 113 113 113 F F F
85502023-12-10T18:53:17.509Z 14 280-299 63 63 63 106 106 106 F F F
85512023-12-10T18:53:17.509Z 15 300-319 63 63 63 106 106 106 F F F
85522023-12-10T18:53:17.509Z 16 320-339 64 64 64 110 110 110 F F F
85532023-12-10T18:53:17.509Z 17 340-359 68 68 68 117 117 117 F F F
85542023-12-10T18:53:17.510Z 18 360-379 68 68 68 117 117 117 F F F
85552023-12-10T18:53:17.510Z 19 380-399 68 68 68 117 117 117 F F F
85562023-12-10T18:53:17.510Z 20 400-419 68 68 68 118 118 118 F F F
85572023-12-10T18:53:17.510Z 21 420-439 59 59 59 98 98 98 F F F
85582023-12-10T18:53:17.510Z 22 440-459 66 66 66 115 115 115 F F F
85592023-12-10T18:53:17.510Z 23 460-479 65 65 65 113 113 113 F F F
85602023-12-10T18:53:17.510Z 24 480-499 66 66 66 114 114 114 F F F
85612023-12-10T18:53:17.510Z 25 500-519 68 68 68 117 117 117 F F F
85622023-12-10T18:53:17.511Z 26 520-539 64 64 64 107 107 107 F F F
85632023-12-10T18:53:17.511Z 27 540-559 68 68 68 120 120 120 F F F
85642023-12-10T18:53:17.511Z 28 560-579 68 68 68 120 120 120 F F F
85652023-12-10T18:53:17.511Z 29 580-599 64 64 64 111 111 111 F F F
85662023-12-10T18:53:17.512ZMax gen: 69, Max flush: 121
85672023-12-10T18:53:17.512Z
85682023-12-10T18:53:17.512ZBegin loop 35
85692023-12-10T18:53:17.512ZDownstairs to restart: 2
85702023-12-10T18:53:17.512Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.203306479Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442}
85712023-12-10T18:53:17.513Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.203763662Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442}
85722023-12-10T18:53:17.514Z{Crucible runtime is spawned
85732023-12-10T18:53:17.514Z"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.203903844Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442}
85742023-12-10T18:53:17.514Z{"msg":"Crucible eff412ea-8f62-4ca5-9244-c67e0584c8b9 has session id: dc073f48-3289-4e43-b8e7-2b79c40c9ef2","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.203944858Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
85752023-12-10T18:53:17.515Z{"msg":"Upstairs opts: Upstairs UUID: eff412ea-8f62-4ca5-9244-c67e0584c8b9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.204001564Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
85762023-12-10T18:53:17.515Z{"msg":"Crucible stats registered with UUID: eff412ea-8f62-4ca5-9244-c67e0584c8b9","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.204095875Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
85772023-12-10T18:53:17.515Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.204139269Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2","task":"up_listen"}
85782023-12-10T18:53:17.516Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30The guest has requested activation with gen:70
85792023-12-10T18:53:17.516Z,"time":"2023-12-10T18:53:17.204236226Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
85802023-12-10T18:53:17.516Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.204396444Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
85812023-12-10T18:53:17.516Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.204444046Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"looper":"0","session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
85822023-12-10T18:53:17.517Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.204492996Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"looper":"1","session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
85832023-12-10T18:53:17.517Z{"msg":"[0] eff412ea-8f62-4ca5-9244-c67e0584c8b9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.204520517Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"looper":"0","session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
85842023-12-10T18:53:17.517Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.204542416Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"looper":"2","session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
85852023-12-10T18:53:17.518Z{"msg":"[1] eff412ea-8f62-4ca5-9244-c67e0584c8b9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.204730598Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"looper":"1","session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
85862023-12-10T18:53:17.518Z{"msg":"[2] eff412ea-8f62-4ca5-9244-c67e0584c8b9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.204815385Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"looper":"2","session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
85872023-12-10T18:53:17.518Z{"msg":"eff412ea-8f62-4ca5-9244-c67e0584c8b9 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.205060168Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
85882023-12-10T18:53:17.518Z{"msg":"Set desired generation to :70","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.205134938Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
85892023-12-10T18:53:17.518Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.205162527Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
85902023-12-10T18:53:17.518Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.205628688Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
85912023-12-10T18:53:17.519Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.205824055Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
85922023-12-10T18:53:17.519Z{"msg":"[0] eff412ea-8f62-4ca5-9244-c67e0584c8b9 (dc073f48-3289-4e43-b8e7-2b79c40c9ef2) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.205997402Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
85932023-12-10T18:53:17.519Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.206163237Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
85942023-12-10T18:53:17.519Z{"msg":"[0] client is_active_req TRUE, promote! session dc073f48-3289-4e43-b8e7-2b79c40c9ef2","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.206203411Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
85952023-12-10T18:53:17.519Z{"msg":"[1] eff412ea-8f62-4ca5-9244-c67e0584c8b9 (dc073f48-3289-4e43-b8e7-2b79c40c9ef2) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.206271761Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
85962023-12-10T18:53:17.520Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.20631576Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
85972023-12-10T18:53:17.520Z{"msg":"[1] client is_active_req TRUE, promote! session dc073f48-3289-4e43-b8e7-2b79c40c9ef2","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.206393947Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
85982023-12-10T18:53:17.520Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.206571735Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
85992023-12-10T18:53:17.520Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.206657731Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86002023-12-10T18:53:17.520Z{"msg":"eff412ea-8f62-4ca5-9244-c67e0584c8b9 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.206753787Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86012023-12-10T18:53:17.520Z{"msg":"[2] eff412ea-8f62-4ca5-9244-c67e0584c8b9 (dc073f48-3289-4e43-b8e7-2b79c40c9ef2) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.206898665Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86022023-12-10T18:53:17.522Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.206942252Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86032023-12-10T18:53:17.522Z{"msg":"[2] client is_active_req TRUE, promote! session dc073f48-3289-4e43-b8e7-2b79c40c9ef2","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.206979961Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86042023-12-10T18:53:17.522Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.207048485Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86052023-12-10T18:53:17.522Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.207080369Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86062023-12-10T18:53:17.523Z{"msg":"eff412ea-8f62-4ca5-9244-c67e0584c8b9 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.207101653Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86072023-12-10T18:53:17.523Z{"msg":"[0] eff412ea-8f62-4ca5-9244-c67e0584c8b9 (dc073f48-3289-4e43-b8e7-2b79c40c9ef2) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.207421917Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86082023-12-10T18:53:17.523Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.207488804Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86092023-12-10T18:53:17.523Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:17.2075302Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86102023-12-10T18:53:17.523Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.207567252Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86112023-12-10T18:53:17.524Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.207590899Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86122023-12-10T18:53:17.524Z{"msg":"eff412ea-8f62-4ca5-9244-c67e0584c8b9 WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.207612336Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86132023-12-10T18:53:17.524Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.207644113Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86142023-12-10T18:53:17.524Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.207679849Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86152023-12-10T18:53:17.524Z{"msg":"[1] eff412ea-8f62-4ca5-9244-c67e0584c8b9 (dc073f48-3289-4e43-b8e7-2b79c40c9ef2) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.207892199Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86162023-12-10T18:53:17.524Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.207956838Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86172023-12-10T18:53:17.524Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:17.207982205Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86182023-12-10T18:53:17.525Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.208002422Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86192023-12-10T18:53:17.525Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.208045171Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86202023-12-10T18:53:17.525Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.208073255Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86212023-12-10T18:53:17.525Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.208115048Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86222023-12-10T18:53:17.525Z{"msg":"eff412ea-8f62-4ca5-9244-c67e0584c8b9 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.208138406Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86232023-12-10T18:53:17.525Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.208179234Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86242023-12-10T18:53:17.525Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.208201312Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86252023-12-10T18:53:17.525Z{"msg":"eff412ea-8f62-4ca5-9244-c67e0584c8b9 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.20822254Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86262023-12-10T18:53:17.526Z{"msg":"[2] eff412ea-8f62-4ca5-9244-c67e0584c8b9 (dc073f48-3289-4e43-b8e7-2b79c40c9ef2) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.208572243Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86272023-12-10T18:53:17.526Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.208626059Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86282023-12-10T18:53:17.526Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:17.208654779Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86292023-12-10T18:53:17.526Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.20869101Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86302023-12-10T18:53:17.526Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.208751276Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86312023-12-10T18:53:17.527Z{"msg":"eff412ea-8f62-4ca5-9244-c67e0584c8b9 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.208780573Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86322023-12-10T18:53:17.527Z{"msg":"[0]R flush_numbers[0..12]: [116, 116, 113, 119, 117, 117, 112, 112, 104, 117, 117, 116]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.208838626Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86332023-12-10T18:53:17.528Z{"msg":"[0]R generation[0..12]: [67, 67, 65, 68, 68, 68, 64, 64, 62, 68, 68, 67]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.20887076Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86342023-12-10T18:53:17.528Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.208906785Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86352023-12-10T18:53:17.595Z{"msg":"[1]R flush_numbers[0..12]: [116, 116, 113, 119, 117, 117, 112, 112, 104, 117, 117, 116]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.208937382Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":The guest has finished waiting for activation with:70
86362023-12-10T18:53:17.595Z1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86372023-12-10T18:53:17.595Z{"msg":"[1]R generation[0..12]: [67, 67, 65, 68, 68, 68, 64, 64, 62, 68, 68, 67]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.209071125Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid"Wait for a query_work_queue command to finish before sending IO
86382023-12-10T18:53:17.595Z:1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86392023-12-10T18:53:17.595Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.209188716Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86402023-12-10T18:53:17.595Z{"msg":"[2]R flush_numbers[0..12]: [116, 116, 113, 119, 117, 117, 112, 112, 104, 117, 117, 116]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.209279428Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86412023-12-10T18:53:17.595Z{"msg":"[2]R generation[0..12]: [67, 67, 65, 68, 68, 68, 64, 64, 62, 68, 68, 67]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.209307143Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86422023-12-10T18:53:17.595Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.20936642Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86432023-12-10T18:53:17.596Z{"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.209391479Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86442023-12-10T18:53:17.596Z{"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.20945272Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86452023-12-10T18:53:17.596Z{"msg":"Next flush: 122","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.209477748Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86462023-12-10T18:53:17.596Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.209501684Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86472023-12-10T18:53:17.596Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.20955566Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86482023-12-10T18:53:17.596Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.209580085Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86492023-12-10T18:53:17.596Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.209636087Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86502023-12-10T18:53:17.596Z{"msg":"eff412ea-8f62-4ca5-9244-c67e0584c8b9 is now active with session: dc073f48-3289-4e43-b8e7-2b79c40c9ef2","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.209668122Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86512023-12-10T18:53:17.596Z{"msg":"eff412ea-8f62-4ca5-9244-c67e0584c8b9 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.209709447Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86522023-12-10T18:53:17.596Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.209747943Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86532023-12-10T18:53:17.596Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.209809622Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86542023-12-10T18:53:17.596Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.209835126Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86552023-12-10T18:53:17.596Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.20991807Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86562023-12-10T18:53:17.596Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.209943453Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86572023-12-10T18:53:17.596Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.209997007Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86582023-12-10T18:53:17.596Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.210021899Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86592023-12-10T18:53:17.596Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.210048402Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86602023-12-10T18:53:17.597Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.21010101Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86612023-12-10T18:53:17.598Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
86622023-12-10T18:53:17.598Z,"time":"2023-12-10T18:53:17.210125144Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86632023-12-10T18:53:17.598Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:17.210261085Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1442,"session_id":"dc073f48-3289-4e43-b8e7-2b79c40c9ef2"}
86642023-12-10T18:53:17.598ZLoading write count information from file "/tmp/test_repair_verify.data"
86652023-12-10T18:53:17.598ZRun Repair workload
86662023-12-10T18:53:17.598Z01/30 Write block 220 len 24576 data: 8 11 11 8 8 6
86672023-12-10T18:53:17.598Z02/30 Write block 98 len 20480 data: 4 5 6 5 7
86682023-12-10T18:53:17.598Z03/30 Write block 451 len 40960 data: 5 5 6 5 6 6 6 7 6 4
86692023-12-10T18:53:17.598Z04/30 Read block 274 len 16384
86702023-12-10T18:53:17.598Z05/30 Read block 176 len 40960
86712023-12-10T18:53:17.598Z06/30 Read block 358 len 24576
86722023-12-10T18:53:17.598Z07/30 Write block 195 len 32768 data: 9 8 9 8 8 7 7 6
86732023-12-10T18:53:17.598Z08/30 Read block 529 len 36864
86742023-12-10T18:53:17.598Z09/30 Read block 546 len 4096
86752023-12-10T18:53:17.598Z10/30 Read block 497 len 20480
86762023-12-10T18:53:17.598Z11/30 Write block 309 len 4096 data: 5
86772023-12-10T18:53:17.598Z12/30 Write block 115 len 36864 data: 5 7 8 8 9 10 10 10 12
86782023-12-10T18:53:17.599Z13/30 Write block 584 len 36864 data: 5 3 4 5 5 5 5 5 4
86792023-12-10T18:53:17.599Z14/30 Read block 256 len 32768
86802023-12-10T18:53:17.599Z15/30 Read block 242 len 4096
86812023-12-10T18:53:17.599Z16/30 Write block 457 len 8192 data: 7 8
86822023-12-10T18:53:17.599Z17/30 Read block 189 len 28672
86832023-12-10T18:53:17.599Z18/30 Read block 346 len 28672
86842023-12-10T18:53:17.599Z19/30 Read block 363 len 28672
86852023-12-10T18:53:17.599Z20/30 Read block 315 len 4096
86862023-12-10T18:53:17.599Z21/30 Read block 397 len 12288
86872023-12-10T18:53:17.599Z22/30 Read block 377 len 16384
86882023-12-10T18:53:17.599Z23/30 Read block 192 len 24576
86892023-12-10T18:53:17.599Z24/30 Write block 358 len 24576 data: 6 6 6 5 5 5
86902023-12-10T18:53:17.599Z25/30 Read block 410 len 24576
86912023-12-10T18:53:17.599Z26/30 Read block 301 len 40960
86922023-12-10T18:53:17.600Z27/30 Read block 369 len 40960
86932023-12-10T18:53:17.600Z28/30 Write block 14 len 8192 data: 10 9
86942023-12-10T18:53:17.600Z29/30 Read block 34 len 16384
86952023-12-10T18:53:17.600Z30/30 Read block 421 len 36864
86962023-12-10T18:53:17.600Z----------------------------------------------------------------
86972023-12-10T18:53:17.600Z Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30
86982023-12-10T18:53:17.600ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
86992023-12-10T18:53:17.600Z 1 Acked 1000 Write 6 Done Done Done false
87002023-12-10T18:53:17.600Z 2 Acked 1001 Write 5 Done Done Done false
87012023-12-10T18:53:17.600Z 3 Acked 1002 Write 10 Done Done Done false
87022023-12-10T18:53:17.600Z 4 Acked 1003 Read 4 Done Done Done false
87032023-12-10T18:53:17.600Z 5 Acked 1004 Read 10 Done Done Sent false
87042023-12-10T18:53:17.600Z 6 Acked 1005 Read 6 Done Done Sent false
87052023-12-10T18:53:17.600Z 7 Acked 1006 Write 8 Done Done Sent false
87062023-12-10T18:53:17.600Z 8 Acked 1007 Read 9 Done Done Sent false
87072023-12-10T18:53:17.600Z 9 Acked 1008 Read 1 Done Done Sent false
87082023-12-10T18:53:17.600Z 10 Acked 1009 Read 5 Done Done Sent false
87092023-12-10T18:53:17.600Z 11 Acked 1010 Write 1 Done Done Sent false
87102023-12-10T18:53:17.601Z 12 Acked 1011 Write 9 Done Done Sent false
87112023-12-10T18:53:17.601Z 13 Acked 1012 Write 9 Done Done Sent false
87122023-12-10T18:53:17.601Z 14 Acked 1013 Read 8 Done Done Sent false
87132023-12-10T18:53:17.601Z 15 Acked 1014 Read 1 Done Done Sent false
87142023-12-10T18:53:17.601Z 16 Acked 1015 Write 2 Done Done Sent false
87152023-12-10T18:53:17.601Z 17 Acked 1016 Read 7 Done Done Sent false
87162023-12-10T18:53:17.601Z 18 Acked 1017 Read 7 Done Done Sent false
87172023-12-10T18:53:17.601Z 19 Acked 1018 Read 7 Done Done Sent false
87182023-12-10T18:53:17.601Z 20 Acked 1019 Read 1 Done Done Sent false
87192023-12-10T18:53:17.601Z 21 Acked 1020 Read 3 Done Done Sent false
87202023-12-10T18:53:17.601Z 22 Acked 1021 Read 4 Done Done Sent false
87212023-12-10T18:53:17.601Z 23 Acked 1022 Read 6 Done Done Sent false
87222023-12-10T18:53:17.601Z 24 Acked 1023 Write 6 Done Done Sent false
87232023-12-10T18:53:17.601Z 25 Acked 1024 Read 6 Done Done Sent false
87242023-12-10T18:53:17.601Z 26 Acked 1025 Read 10 Done Done Sent false
87252023-12-10T18:53:17.601Z 27 Acked 1026 Read 10 Done Done Sent false
87262023-12-10T18:53:17.601Z 28 Acked 1027 Write 2 Done Done Sent false
87272023-12-10T18:53:17.601Z 29 Acked 1028 Read 4 Done Done Sent false
87282023-12-10T18:53:17.602Z 30 Acked 1029 Read 9 Done Sent Sent false
87292023-12-10T18:53:17.602Z STATES DS:0 DS:1 DS:2 TOTAL
87302023-12-10T18:53:17.602Z New 0 0 0 0
87312023-12-10T18:53:17.602Z Sent 0 1 26 27
87322023-12-10T18:53:17.602Z Done 30 29 4 63
87332023-12-10T18:53:17.602Z Skipped 0 0 0 0
87342023-12-10T18:53:17.602Z Error 0 0 0 0
87352023-12-10T18:53:17.646ZLast Flush: 0 0 0
87362023-12-10T18:53:17.646ZDownstairs last five completed:
87372023-12-10T18:53:17.646ZUpstairs last five completed: 30 29 28 27 26 25
87382023-12-10T18:53:17.647ZWrote out file "/tmp/test_repair_verify.data"
87392023-12-10T18:53:17.647Z
87402023-12-10T18:53:17.647ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
87412023-12-10T18:53:17.647ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
87422023-12-10T18:53:17.647Z 0 000-019 67 67 67 116 116 116 T T F <---
87432023-12-10T18:53:17.647Z 1 020-039 67 67 67 116 116 116 F F F
87442023-12-10T18:53:17.647Z 2 040-059 65 65 65 113 113 113 F F F
87452023-12-10T18:53:17.647Z 3 060-079 68 68 68 119 119 119 F F F
87462023-12-10T18:53:17.647Z 4 080-099 68 68 68 117 117 117 T T T
87472023-12-10T18:53:17.647Z 5 100-119 68 68 68 117 117 117 T T T
87482023-12-10T18:53:17.648Z 6 120-139 64 64 64 112 112 112 T T F <---
87492023-12-10T18:53:17.648Z 7 140-159 64 64 64 112 112 112 F F F
87502023-12-10T18:53:17.648Z 8 160-179 62 62 62 104 104 104 F F F
87512023-12-10T18:53:17.648Z 9 180-199 68 68 68 117 117 117 T T F <---
87522023-12-10T18:53:17.648Z 10 200-219 68 68 68 117 117 117 T T F <---
87532023-12-10T18:53:17.648Z 11 220-239 67 67 67 116 116 116 T T T
87542023-12-10T18:53:17.648Z 12 240-259 69 69 69 121 121 121 F F F
87552023-12-10T18:53:17.648Z 13 260-279 65 65 65 113 113 113 F F F
87562023-12-10T18:53:17.648Z 14 280-299 63 63 63 106 106 106 F F F
87572023-12-10T18:53:17.648Z 15 300-319 63 63 63 106 106 106 T T F <---
87582023-12-10T18:53:17.649Z 16 320-339 64 64 64 110 110 110 F F F
87592023-12-10T18:53:17.649Z 17 340-359 68 68 68 117 117 117 T T F <---
87602023-12-10T18:53:17.649Z 18 360-379 68 68 68 117 117 117 T T F <---
87612023-12-10T18:53:17.650Z 19 380-399 68 68 68 117 117 117 F F F
87622023-12-10T18:53:17.650Z 20 400-419 68 68 68 118 118 118 F F F
87632023-12-10T18:53:17.650Z 21 420-439 59 59 59 98 98 98 F F F
87642023-12-10T18:53:17.650Z 22 440-459 66 66 66 115 115 115 T T T
87652023-12-10T18:53:17.650Z 23 460-479 65 65 65 113 113 113 T T T
87662023-12-10T18:53:17.650Z 24 480-499 66 66 66 114 114 114 F F F
87672023-12-10T18:53:17.650Z 25 500-519 68 68 68 117 117 117 F F F
87682023-12-10T18:53:17.650Z 26 520-539 64 64 64 107 107 107 F F F
87692023-12-10T18:53:17.650Z 27 540-559 68 68 68 120 120 120 F F F
87702023-12-10T18:53:17.650Z 28 560-579 68 68 68 120 120 120 F F F
87712023-12-10T18:53:17.650Z 29 580-599 64 64 64 111 111 111 T T F <---
87722023-12-10T18:53:17.650ZMax gen: 69, Max flush: 121
87732023-12-10T18:53:17.651ZError: Difference in extent metadata found!
87742023-12-10T18:53:17.651ZA Difference in extent metadata is expected here
87752023-12-10T18:53:17.651ZOn loop 35
87762023-12-10T18:53:17.651Z
87772023-12-10T18:53:17.651ZVerifying data now
87782023-12-10T18:53:23.623ZNo change after verify
87792023-12-10T18:53:23.623ZLoop: 35 Downstairs dump after verify (and repair):
87802023-12-10T18:53:23.669ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
87812023-12-10T18:53:23.669Z 0 000-019 70 70 70 122 122 122 F F F
87822023-12-10T18:53:23.673Z 1 020-039 67 67 67 116 116 116 F F F
87832023-12-10T18:53:23.673Z 2 040-059 65 65 65 113 113 113 F F F
87842023-12-10T18:53:23.673Z 3 060-079 68 68 68 119 119 119 F F F
87852023-12-10T18:53:23.673Z 4 080-099 70 70 70 122 122 122 F F F
87862023-12-10T18:53:23.673Z 5 100-119 70 70 70 122 122 122 F F F
87872023-12-10T18:53:23.673Z 6 120-139 70 70 70 122 122 122 F F F
87882023-12-10T18:53:23.673Z 7 140-159 64 64 64 112 112 112 F F F
87892023-12-10T18:53:23.674Z 8 160-179 62 62 62 104 104 104 F F F
87902023-12-10T18:53:23.674Z 9 180-199 70 70 70 122 122 122 F F F
87912023-12-10T18:53:23.674Z 10 200-219 70 70 70 122 122 122 F F F
87922023-12-10T18:53:23.674Z 11 220-239 70 70 70 122 122 122 F F F
87932023-12-10T18:53:23.674Z 12 240-259 69 69 69 121 121 121 F F F
87942023-12-10T18:53:23.674Z 13 260-279 65 65 65 113 113 113 F F F
87952023-12-10T18:53:23.674Z 14 280-299 63 63 63 106 106 106 F F F
87962023-12-10T18:53:23.674Z 15 300-319 70 70 70 122 122 122 F F F
87972023-12-10T18:53:23.675Z 16 320-339 64 64 64 110 110 110 F F F
87982023-12-10T18:53:23.675Z 17 340-359 70 70 70 122 122 122 F F F
87992023-12-10T18:53:23.675Z 18 360-379 70 70 70 122 122 122 F F F
88002023-12-10T18:53:23.675Z 19 380-399 68 68 68 117 117 117 F F F
88012023-12-10T18:53:23.675Z 20 400-419 68 68 68 118 118 118 F F F
88022023-12-10T18:53:23.675Z 21 420-439 59 59 59 98 98 98 F F F
88032023-12-10T18:53:23.675Z 22 440-459 70 70 70 122 122 122 F F F
88042023-12-10T18:53:23.675Z 23 460-479 70 70 70 122 122 122 F F F
88052023-12-10T18:53:23.675Z 24 480-499 66 66 66 114 114 114 F F F
88062023-12-10T18:53:23.675Z 25 500-519 68 68 68 117 117 117 F F F
88072023-12-10T18:53:23.675Z 26 520-539 64 64 64 107 107 107 F F F
88082023-12-10T18:53:23.676Z 27 540-559 68 68 68 120 120 120 F F F
88092023-12-10T18:53:23.676Z 28 560-579 68 68 68 120 120 120 F F F
88102023-12-10T18:53:23.676Z 29 580-599 70 70 70 122 122 122 F F F
88112023-12-10T18:53:23.676ZMax gen: 70, Max flush: 122
88122023-12-10T18:53:23.676Z
88132023-12-10T18:53:23.676ZBegin loop 36
88142023-12-10T18:53:23.676ZDownstairs to restart: 2
88152023-12-10T18:53:23.681Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.683553165Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450}
88162023-12-10T18:53:23.681Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.683982265Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450}
88172023-12-10T18:53:23.681Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.684044625Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450}
88182023-12-10T18:53:23.691Z{"msg":"Crucible fac3688e-bf63-4227-8090-7323e4f2c50b has session id: 07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.684070558Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88192023-12-10T18:53:23.691Z{"msg":"Upstairs opts: Upstairs UUID: fac3688e-bf63-4227-8090-7323e4f2c50b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.684138107Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88202023-12-10T18:53:23.691Z{"msg":"Crucible stats registered with UUID: fac3688e-bf63-4227-8090-7323e4f2c50b","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.684184535Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88212023-12-10T18:53:23.691Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.684437141Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"looper":"0","session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88222023-12-10T18:53:23.691ZCrucible runtime is spawned
88232023-12-10T18:53:23.691ZThe guest has requested activation with gen:72
88242023-12-10T18:53:23.692Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.684933653Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"looper":"1","session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88252023-12-10T18:53:23.692Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.685037482Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"looper":"2","session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88262023-12-10T18:53:23.692Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.685078383Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e","task":"up_listen"}
88272023-12-10T18:53:23.692Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.685165227Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88282023-12-10T18:53:23.692Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.685201823Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88292023-12-10T18:53:23.692Z{"msg":"[1] fac3688e-bf63-4227-8090-7323e4f2c50b looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.685235921Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"looper":"1","session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88302023-12-10T18:53:23.692Z{"msg":"fac3688e-bf63-4227-8090-7323e4f2c50b active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.685302903Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88312023-12-10T18:53:23.693Z{"msg":"[0] fac3688e-bf63-4227-8090-7323e4f2c50b looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.685371862Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"looper":"0","session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88322023-12-10T18:53:23.693Z{"msg":"Set desired generation to :72","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.68544118Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88332023-12-10T18:53:23.693Z{"msg":"[2] fac3688e-bf63-4227-8090-7323e4f2c50b looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.685629225Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"looper":"2","session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88342023-12-10T18:53:23.693Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.685779739Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88352023-12-10T18:53:23.693Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.685826624Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88362023-12-10T18:53:23.693Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.685863112Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88372023-12-10T18:53:23.693Z{"msg":"[0] fac3688e-bf63-4227-8090-7323e4f2c50b (07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.685980068Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88382023-12-10T18:53:23.693Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.686086155Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88392023-12-10T18:53:23.694Z{"msg":"[0] client is_active_req TRUE, promote! session 07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.686172207Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88402023-12-10T18:53:23.694Z{"msg":"[1] fac3688e-bf63-4227-8090-7323e4f2c50b (07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.686212073Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88412023-12-10T18:53:23.694Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.686341363Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88422023-12-10T18:53:23.694Z{"msg":"[1] client is_active_req TRUE, promote! session 07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.686388703Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88432023-12-10T18:53:23.694Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.686731479Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88442023-12-10T18:53:23.694Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.686801271Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88452023-12-10T18:53:23.694Z{"msg":"fac3688e-bf63-4227-8090-7323e4f2c50b WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.686826357Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88462023-12-10T18:53:23.694Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.6869286Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88472023-12-10T18:53:23.694Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.686984391Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88482023-12-10T18:53:23.694Z{"msg":"fac3688e-bf63-4227-8090-7323e4f2c50b WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.687008828Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88492023-12-10T18:53:23.694Z{"msg":"[0] fac3688e-bf63-4227-8090-7323e4f2c50b (07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.687504991Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88502023-12-10T18:53:23.694Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.687672152Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88512023-12-10T18:53:23.694Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:23.687713599Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88522023-12-10T18:53:23.694Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.687782238Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88532023-12-10T18:53:23.694Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.687854671Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88542023-12-10T18:53:23.694Z{"msg":"fac3688e-bf63-4227-8090-7323e4f2c50b WaitQuorum WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.687895198Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88552023-12-10T18:53:23.694Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.687962636Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88562023-12-10T18:53:23.695Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.68803132Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88572023-12-10T18:53:23.695Z{"msg":"[1] fac3688e-bf63-4227-8090-7323e4f2c50b (07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e) WaitQuorum WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.688101247Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88582023-12-10T18:53:23.695Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.688168342Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88592023-12-10T18:53:23.695Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:23.688237478Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88602023-12-10T18:53:23.696Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.68827688Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88612023-12-10T18:53:23.696Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.688362509Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88622023-12-10T18:53:23.696Z{"msg":"fac3688e-bf63-4227-8090-7323e4f2c50b WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.68840367Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88632023-12-10T18:53:23.696Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.688442512Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88642023-12-10T18:53:23.696Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.688499971Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88652023-12-10T18:53:23.696Z{"msg":"[2] fac3688e-bf63-4227-8090-7323e4f2c50b (07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.688527852Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88662023-12-10T18:53:23.696Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.688630303Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88672023-12-10T18:53:23.696Z{"msg":"[2] client is_active_req TRUE, promote! session 07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.688657357Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88682023-12-10T18:53:23.703Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.68934456Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88692023-12-10T18:53:23.703Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.689402932Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88702023-12-10T18:53:23.703Z{"msg":"fac3688e-bf63-4227-8090-7323e4f2c50b WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.689427885Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88712023-12-10T18:53:23.703Z{"msg":"[2] fac3688e-bf63-4227-8090-7323e4f2c50b (07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.689977447Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88722023-12-10T18:53:23.703Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.690030514Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88732023-12-10T18:53:23.703Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:23.690054047Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88742023-12-10T18:53:23.703Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.69007393Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88752023-12-10T18:53:23.703Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.690094695Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88762023-12-10T18:53:23.703Z{"msg":"fac3688e-bf63-4227-8090-7323e4f2c50b WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.690116886Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88772023-12-10T18:53:23.703Z{"msg":"[0]R flush_numbers[0..12]: [122, 116, 113, 119, 122, 122, 122, 112, 104, 122, 122, 122]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.690138756Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88782023-12-10T18:53:23.704Z{"msg":"[0]R generation[0..12]: [70, 67, 65, 68, 70, 70, 70, 64, 62, 70, 70, 70]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.690180396Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88792023-12-10T18:53:23.704Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.690204828Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88802023-12-10T18:53:23.704Z{"msg":"[1]R flush_numbers[0..12]: [122, 116, 113, 119, 122, 122, 122, 112, 104, 122, 122, 122]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.690246848Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88812023-12-10T18:53:23.704Z{"msg":"[1]R generation[0..12]: [70, 67, 65, 68, 70, 70, 70, 64, 62, 70, 70, 70]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.690270017Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88822023-12-10T18:53:23.705Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.690316565Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88832023-12-10T18:53:23.705Z{"msg":"[2]R flush_numbers[0..12]: [122, 116, 113, 119, 122, 122, 122, 112, 104, 122, 122, 122]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.690339877Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88842023-12-10T18:53:23.705Z{"msg":"[2]R generation[0..12]: [70, 67, 65, 68, 70, 70, 70, 64, 62, 70, 70, 70]","v":The guest has finished waiting for activation with:72
88852023-12-10T18:53:23.705Z0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
88862023-12-10T18:53:23.705Z,"time":"2023-12-10T18:53:23.690366415Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88872023-12-10T18:53:23.705Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.690390229Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88882023-12-10T18:53:23.705Z{"msg":"Max found gen is 71","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.690431951Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88892023-12-10T18:53:23.706Z{"msg":"Generation requested: 72 >= found:71","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.6905186Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88902023-12-10T18:53:23.706Z{"msg":"Next flush: 123","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.690602234Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88912023-12-10T18:53:23.706Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.690642639Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88922023-12-10T18:53:23.706Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.690707012Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88932023-12-10T18:53:23.706Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.690730856Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88942023-12-10T18:53:23.706Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.690782601Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88952023-12-10T18:53:23.706Z{"msg":"fac3688e-bf63-4227-8090-7323e4f2c50b is now active with session: 07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.690822813Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88962023-12-10T18:53:23.706Z{"msg":"fac3688e-bf63-4227-8090-7323e4f2c50b Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.69086972Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88972023-12-10T18:53:23.706Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.690892831Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88982023-12-10T18:53:23.706Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.690948278Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
88992023-12-10T18:53:23.706Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.690973631Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
89002023-12-10T18:53:23.706Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.691049931Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
89012023-12-10T18:53:23.706Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.69107826Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
89022023-12-10T18:53:23.706Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.691155113Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
89032023-12-10T18:53:23.707Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.691224382Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
89042023-12-10T18:53:23.707Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.691265302Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
89052023-12-10T18:53:23.707Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.691338292Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
89062023-12-10T18:53:23.707Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.691393402Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
89072023-12-10T18:53:23.707Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:23.691416296Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1450,"session_id":"07c4e6f7-c8d6-4c02-b60d-e2748d01aa1e"}
89082023-12-10T18:53:23.707ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
89092023-12-10T18:53:23.707ZLoading write count information from file "/tmp/test_repair_verify.data"
89102023-12-10T18:53:23.707ZRun Repair workload
89112023-12-10T18:53:23.707Z01/30 Flush
89122023-12-10T18:53:23.707Z02/30 Write block 190 len 36864 data: 6 7 7 7 7 10 9 10 9
89132023-12-10T18:53:23.707Z03/30 Read block 91 len 12288
89142023-12-10T18:53:23.707Z04/30 Flush
89152023-12-10T18:53:23.708Z05/30 Write block 479 len 4096 data: 3
89162023-12-10T18:53:23.708Z06/30 Read block 487 len 12288
89172023-12-10T18:53:23.708Z07/30 Write block 15 len 4096 data: 10
89182023-12-10T18:53:23.708Z08/30 Flush
89192023-12-10T18:53:23.711Z09/30 Write block 499 len 24576 data: 5 8 8 8 9 6
89202023-12-10T18:53:23.712Z10/30 Read block 370 len 28672
89212023-12-10T18:53:23.712Z11/30 Flush
89222023-12-10T18:53:23.712Z12/30 Write block 415 len 8192 data: 3 3
89232023-12-10T18:53:23.712Z13/30 Write block 550 len 28672 data: 4 4 4 4 4 5 4
89242023-12-10T18:53:23.712Z14/30 Read block 268 len 32768
89252023-12-10T18:53:23.715Z15/30 Read block 99 len 4096
89262023-12-10T18:53:23.715Z16/30 Write block 374 len 12288 data: 5 7 7
89272023-12-10T18:53:23.715Z17/30 Read block 215 len 28672
89282023-12-10T18:53:23.730Z18/30 Read block 358 len 16384
89292023-12-10T18:53:23.739Z19/30 Write block 193 len 12288 data: 8 8 11
89302023-12-10T18:53:23.739Z20/30 Flush
89312023-12-10T18:53:23.739Z21/30 Write block 146 len 4096 data: 6
89322023-12-10T18:53:23.739Z22/30 Write block 543 len 28672 data: 11 10 6 7 7 8 5
89332023-12-10T18:53:23.739Z23/30 Write block 263 len 16384 data: 4 5 6 5
89342023-12-10T18:53:23.739Z24/30 Write block 373 len 24576 data: 5 6 8 8 7 10
89352023-12-10T18:53:23.739Z25/30 Read block 23 len 36864
89362023-12-10T18:53:23.739Z26/30 Read block 306 len 16384
89372023-12-10T18:53:23.739Z27/30 Write block 283 len 16384 data: 5 7 9 9
89382023-12-10T18:53:23.739Z28/30 Read block 173 len 36864
89392023-12-10T18:53:23.748Z29/30 Read block 199 len 36864
89402023-12-10T18:53:23.748Z30/30 Write block 588 len 12288 data: 6 6 6
89412023-12-10T18:53:23.748Z----------------------------------------------------------------
89422023-12-10T18:53:23.749Z Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30
89432023-12-10T18:53:23.749ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
89442023-12-10T18:53:23.749Z 1 Acked 1000 Flush 0 Done Done Sent false
89452023-12-10T18:53:23.749Z 2 Acked 1001 Write 9 Done Done Sent false
89462023-12-10T18:53:23.749Z 3 Acked 1002 Read 3 Done Done Sent false
89472023-12-10T18:53:23.749Z 4 Acked 1003 Flush 0 Done Done Sent false
89482023-12-10T18:53:23.749Z 5 Acked 1004 Write 1 Done Done Sent false
89492023-12-10T18:53:23.749Z 6 Acked 1005 Read 3 Done Done Sent false
89502023-12-10T18:53:23.749Z 7 Acked 1006 Write 1 Done Done Sent false
89512023-12-10T18:53:23.750Z 8 Acked 1007 Flush 0 Done Done Sent false
89522023-12-10T18:53:23.750Z 9 Acked 1008 Write 6 Done Done Sent false
89532023-12-10T18:53:23.750Z 10 Acked 1009 Read 7 Done Done Sent false
89542023-12-10T18:53:23.750Z 11 Acked 1010 Flush 0 Done Done Sent false
89552023-12-10T18:53:23.750Z 12 Acked 1011 Write 2 Done Done Sent false
89562023-12-10T18:53:23.750Z 13 Acked 1012 Write 7 Done Done Sent false
89572023-12-10T18:53:23.750Z 14 Acked 1013 Read 8 Done Done Sent false
89582023-12-10T18:53:23.750Z 15 Acked 1014 Read 1 Done Done Sent false
89592023-12-10T18:53:23.750Z 16 Acked 1015 Write 3 Done Done Sent false
89602023-12-10T18:53:23.750Z 17 Acked 1016 Read 7 Done Done Sent false
89612023-12-10T18:53:23.751Z 18 Acked 1017 Read 4 Done Done Sent false
89622023-12-10T18:53:23.751Z 19 Acked 1018 Write 3 Done Done Sent false
89632023-12-10T18:53:23.751Z 20 Acked 1019 Flush 0 Done Done Sent false
89642023-12-10T18:53:23.751Z 21 Acked 1020 Write 1 Done Done Sent false
89652023-12-10T18:53:23.751Z 22 Acked 1021 Write 7 Done Done Sent false
89662023-12-10T18:53:23.751Z 23 Acked 1022 Write 4 Done Done Sent false
89672023-12-10T18:53:23.751Z 24 Acked 1023 Write 6 Done Done Sent false
89682023-12-10T18:53:23.751Z 25 Acked 1024 Read 9 Done Done Sent false
89692023-12-10T18:53:23.751Z 26 Acked 1025 Read 4 Done Done Sent false
89702023-12-10T18:53:23.752Z 27 Acked 1026 Write 4 Done Done Sent false
89712023-12-10T18:53:23.752Z 28 Acked 1027 Read 9 Done Done Sent false
89722023-12-10T18:53:23.752Z 29 Acked 1028 Read 9 Done Done Sent false
89732023-12-10T18:53:23.752Z 30 Acked 1029 Write 3 Sent Sent Sent false
89742023-12-10T18:53:23.752Z STATES DS:0 DS:1 DS:2 TOTAL
89752023-12-10T18:53:23.752Z New 0 0 0 0
89762023-12-10T18:53:23.752Z Sent 1 1 30 32
89772023-12-10T18:53:23.752Z Done 29 29 0 58
89782023-12-10T18:53:23.752Z Skipped 0 0 0 0
89792023-12-10T18:53:23.752Z Error 0 0 0 0
89802023-12-10T18:53:23.753ZLast Flush: 1019 1019 0
89812023-12-10T18:53:23.753ZDownstairs last five completed:
89822023-12-10T18:53:23.753ZUpstairs last five completed: 30 29 28 27 26 25
89832023-12-10T18:53:23.753ZWrote out file "/tmp/test_repair_verify.data"
89842023-12-10T18:53:23.760Z
89852023-12-10T18:53:23.760ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
89862023-12-10T18:53:23.779ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
89872023-12-10T18:53:23.779Z 0 000-019 72 72 70 125 125 122 F F F <---
89882023-12-10T18:53:23.779Z 1 020-039 67 67 67 116 116 116 F F F
89892023-12-10T18:53:23.779Z 2 040-059 65 65 65 113 113 113 F F F
89902023-12-10T18:53:23.779Z 3 060-079 68 68 68 119 119 119 F F F
89912023-12-10T18:53:23.799Z 4 080-099 70 70 70 122 122 122 F F F
89922023-12-10T18:53:23.799Z 5 100-119 70 70 70 122 122 122 F F F
89932023-12-10T18:53:23.800Z 6 120-139 70 70 70 122 122 122 F F F
89942023-12-10T18:53:23.800Z 7 140-159 64 64 64 112 112 112 T T F <---
89952023-12-10T18:53:23.800Z 8 160-179 62 62 62 104 104 104 F F F
89962023-12-10T18:53:23.800Z 9 180-199 72 72 70 127 127 122 F F F <---
89972023-12-10T18:53:23.800Z 10 200-219 70 70 70 122 122 122 F F F
89982023-12-10T18:53:23.800Z 11 220-239 70 70 70 122 122 122 F F F
89992023-12-10T18:53:23.800Z 12 240-259 69 69 69 121 121 121 F F F
90002023-12-10T18:53:23.800Z 13 260-279 65 65 65 113 113 113 T T F <---
90012023-12-10T18:53:23.800Z 14 280-299 63 63 63 106 106 106 T T F <---
90022023-12-10T18:53:23.800Z 15 300-319 70 70 70 122 122 122 F F F
90032023-12-10T18:53:23.801Z 16 320-339 64 64 64 110 110 110 F F F
90042023-12-10T18:53:23.801Z 17 340-359 70 70 70 122 122 122 F F F
90052023-12-10T18:53:23.801Z 18 360-379 72 72 70 127 127 122 T T F <---
90062023-12-10T18:53:23.801Z 19 380-399 68 68 68 117 117 117 F F F
90072023-12-10T18:53:23.801Z 20 400-419 72 72 68 127 127 118 F F F <---
90082023-12-10T18:53:23.801Z 21 420-439 59 59 59 98 98 98 F F F
90092023-12-10T18:53:23.801Z 22 440-459 70 70 70 122 122 122 F F F
90102023-12-10T18:53:23.801Z 23 460-479 72 72 70 125 125 122 F F F <---
90112023-12-10T18:53:23.801Z 24 480-499 72 72 66 126 126 114 F F F <---
90122023-12-10T18:53:23.801Z 25 500-519 72 72 68 126 126 117 F F F <---
90132023-12-10T18:53:23.802Z 26 520-539 64 64 64 107 107 107 F F F
90142023-12-10T18:53:23.802Z 27 540-559 72 72 68 127 127 120 T T F <---
90152023-12-10T18:53:23.802Z 28 560-579 68 68 68 120 120 120 F F F
90162023-12-10T18:53:23.802Z 29 580-599 70 70 70 122 122 122 T T F <---
90172023-12-10T18:53:23.802ZMax gen: 72, Max flush: 127
90182023-12-10T18:53:23.802ZError: Difference in extent metadata found!
90192023-12-10T18:53:23.803ZA Difference in extent metadata is expected here
90202023-12-10T18:53:23.805ZOn loop 36
90212023-12-10T18:53:23.807Z
90222023-12-10T18:53:23.809ZVerifying data now
90232023-12-10T18:53:29.199ZNo change after verify
90242023-12-10T18:53:29.199ZLoop: 36 Downstairs dump after verify (and repair):
90252023-12-10T18:53:29.208ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
90262023-12-10T18:53:29.239Z 0 000-019 72 72 72 125 125 125 F F F
90272023-12-10T18:53:29.240Z 1 020-039 67 67 67 116 116 116 F F F
90282023-12-10T18:53:29.240Z 2 040-059 65 65 65 113 113 113 F F F
90292023-12-10T18:53:29.240Z 3 060-079 68 68 68 119 119 119 F F F
90302023-12-10T18:53:29.241Z 4 080-099 70 70 70 122 122 122 F F F
90312023-12-10T18:53:29.241Z 5 100-119 70 70 70 122 122 122 F F F
90322023-12-10T18:53:29.241Z 6 120-139 70 70 70 122 122 122 F F F
90332023-12-10T18:53:29.242Z 7 140-159 73 73 73 128 128 128 F F F
90342023-12-10T18:53:29.242Z 8 160-179 62 62 62 104 104 104 F F F
90352023-12-10T18:53:29.242Z 9 180-199 72 72 72 127 127 127 F F F
90362023-12-10T18:53:29.242Z 10 200-219 70 70 70 122 122 122 F F F
90372023-12-10T18:53:29.243Z 11 220-239 70 70 70 122 122 122 F F F
90382023-12-10T18:53:29.243Z 12 240-259 69 69 69 121 121 121 F F F
90392023-12-10T18:53:29.243Z 13 260-279 73 73 73 128 128 128 F F F
90402023-12-10T18:53:29.243Z 14 280-299 73 73 73 128 128 128 F F F
90412023-12-10T18:53:29.243Z 15 300-319 70 70 70 122 122 122 F F F
90422023-12-10T18:53:29.243Z 16 320-339 64 64 64 110 110 110 F F F
90432023-12-10T18:53:29.244Z 17 340-359 70 70 70 122 122 122 F F F
90442023-12-10T18:53:29.244Z 18 360-379 73 73 73 128 128 128 F F F
90452023-12-10T18:53:29.244Z 19 380-399 68 68 68 117 117 117 F F F
90462023-12-10T18:53:29.244Z 20 400-419 72 72 72 127 127 127 F F F
90472023-12-10T18:53:29.244Z 21 420-439 59 59 59 98 98 98 F F F
90482023-12-10T18:53:29.244Z 22 440-459 70 70 70 122 122 122 F F F
90492023-12-10T18:53:29.244Z 23 460-479 72 72 72 125 125 125 F F F
90502023-12-10T18:53:29.244Z 24 480-499 72 72 72 126 126 126 F F F
90512023-12-10T18:53:29.245Z 25 500-519 72 72 72 126 126 126 F F F
90522023-12-10T18:53:29.245Z 26 520-539 64 64 64 107 107 107 F F F
90532023-12-10T18:53:29.245Z 27 540-559 73 73 73 128 128 128 F F F
90542023-12-10T18:53:29.245Z 28 560-579 68 68 68 120 120 120 F F F
90552023-12-10T18:53:29.245Z 29 580-599 73 73 73 128 128 128 F F F
90562023-12-10T18:53:29.245ZMax gen: 73, Max flush: 128
90572023-12-10T18:53:29.246Z
90582023-12-10T18:53:29.246ZBegin loop 37
90592023-12-10T18:53:29.246ZDownstairs to restart: 1
90602023-12-10T18:53:29.246Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.048176526Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458}
90612023-12-10T18:53:29.246Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.048599519Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458}
90622023-12-10T18:53:29.246Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.048659879Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458}
90632023-12-10T18:53:29.246Z{"msg":"Crucible 5cf19ad2-8846-48a0-80f5-34d72e8cb959 has session id: 9804a389-432e-4b33-960f-0bda85c95ad6","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.048685873Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90642023-12-10T18:53:29.246Z{"msg":"Upstairs opts: Upstairs UUID: 5cf19ad2-8846-48a0-80f5-34d72e8cb959, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.048762058Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90652023-12-10T18:53:29.247Z{"msg":"Crucible stats registered with UUID: 5cf19ad2-8846-48a0-80f5-34d72e8cb959","v":0,"name":"crucible","level":30Crucible runtime is spawned
90662023-12-10T18:53:29.248Z,"time":"2023-12-10T18:53:29.048836795Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90672023-12-10T18:53:29.248ZThe guest has requested activation with gen:74
90682023-12-10T18:53:29.248Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.049082258Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"looper":"2","session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90692023-12-10T18:53:29.248Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.049149078Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6","task":"up_listen"}
90702023-12-10T18:53:29.248Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.049197573Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90712023-12-10T18:53:29.248Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.049246934Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90722023-12-10T18:53:29.248Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.049370878Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"looper":"0","session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90732023-12-10T18:53:29.249Z{"msg":"[2] 5cf19ad2-8846-48a0-80f5-34d72e8cb959 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.049504638Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"looper":"2","session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90742023-12-10T18:53:29.249Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.049605487Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"looper":"1","session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90752023-12-10T18:53:29.249Z{"msg":"[0] 5cf19ad2-8846-48a0-80f5-34d72e8cb959 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.049718439Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"looper":"0","session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90762023-12-10T18:53:29.249Z{"msg":"[1] 5cf19ad2-8846-48a0-80f5-34d72e8cb959 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.049787699Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"looper":"1","session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90772023-12-10T18:53:29.249Z{"msg":"5cf19ad2-8846-48a0-80f5-34d72e8cb959 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.050144315Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90782023-12-10T18:53:29.250Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.050247837Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90792023-12-10T18:53:29.250Z{"msg":"Set desired generation to :74","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.05032934Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90802023-12-10T18:53:29.251Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.050574605Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90812023-12-10T18:53:29.251Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.050762703Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90822023-12-10T18:53:29.251Z{"msg":"[2] 5cf19ad2-8846-48a0-80f5-34d72e8cb959 (9804a389-432e-4b33-960f-0bda85c95ad6) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.050875654Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90832023-12-10T18:53:29.251Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.050979492Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90842023-12-10T18:53:29.251Z{"msg":"[2] client is_active_req TRUE, promote! session 9804a389-432e-4b33-960f-0bda85c95ad6","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.051055009Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90852023-12-10T18:53:29.251Z{"msg":"[0] 5cf19ad2-8846-48a0-80f5-34d72e8cb959 (9804a389-432e-4b33-960f-0bda85c95ad6) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.051101304Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90862023-12-10T18:53:29.251Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.051137112Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90872023-12-10T18:53:29.251Z{"msg":"[0] client is_active_req TRUE, promote! session 9804a389-432e-4b33-960f-0bda85c95ad6","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.051233107Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90882023-12-10T18:53:29.252Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.051621592Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90892023-12-10T18:53:29.252Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.051748523Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90902023-12-10T18:53:29.252Z{"msg":"5cf19ad2-8846-48a0-80f5-34d72e8cb959 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.051834767Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90912023-12-10T18:53:29.252Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.051913235Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90922023-12-10T18:53:29.252Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.051996778Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90932023-12-10T18:53:29.252Z{"msg":"5cf19ad2-8846-48a0-80f5-34d72e8cb959 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.052071553Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90942023-12-10T18:53:29.252Z{"msg":"[1] 5cf19ad2-8846-48a0-80f5-34d72e8cb959 (9804a389-432e-4b33-960f-0bda85c95ad6) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.052118665Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90952023-12-10T18:53:29.252Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.052162426Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90962023-12-10T18:53:29.252Z{"msg":"[1] client is_active_req TRUE, promote! session 9804a389-432e-4b33-960f-0bda85c95ad6","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.052186642Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90972023-12-10T18:53:29.262Z{"msg":"[2] 5cf19ad2-8846-48a0-80f5-34d72e8cb959 (9804a389-432e-4b33-960f-0bda85c95ad6) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.052443732Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90982023-12-10T18:53:29.262Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.052493807Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
90992023-12-10T18:53:29.263Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:29.052586708Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91002023-12-10T18:53:29.263Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.052661933Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91012023-12-10T18:53:29.263Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.052720352Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91022023-12-10T18:53:29.263Z{"msg":"5cf19ad2-8846-48a0-80f5-34d72e8cb959 WaitActive WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.052819757Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91032023-12-10T18:53:29.263Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.05290765Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91042023-12-10T18:53:29.264Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.052967367Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91052023-12-10T18:53:29.264Z{"msg":"[0] 5cf19ad2-8846-48a0-80f5-34d72e8cb959 (9804a389-432e-4b33-960f-0bda85c95ad6) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.053029903Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91062023-12-10T18:53:29.264Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.053086325Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91072023-12-10T18:53:29.264Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:29.053126485Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91082023-12-10T18:53:29.264Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.053181891Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91092023-12-10T18:53:29.264Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.053239981Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91102023-12-10T18:53:29.265Z{"msg":"5cf19ad2-8846-48a0-80f5-34d72e8cb959 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.053297435Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91112023-12-10T18:53:29.265Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.053359733Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91122023-12-10T18:53:29.265Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.053415663Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91132023-12-10T18:53:29.265Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.05347596Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91142023-12-10T18:53:29.265Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.053540121Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91152023-12-10T18:53:29.265Z{"msg":"5cf19ad2-8846-48a0-80f5-34d72e8cb959 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.053619184Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91162023-12-10T18:53:29.265Z{"msg":"[1] 5cf19ad2-8846-48a0-80f5-34d72e8cb959 (9804a389-432e-4b33-960f-0bda85c95ad6) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.053666289Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91172023-12-10T18:53:29.267Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.05371072Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91182023-12-10T18:53:29.267Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:29.053827981Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91192023-12-10T18:53:29.267Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.053872377Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91202023-12-10T18:53:29.267Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.053894849Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91212023-12-10T18:53:29.267Z{"msg":"5cf19ad2-8846-48a0-80f5-34d72e8cb959 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.053939081Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91222023-12-10T18:53:29.267Z{The guest has finished waiting for activation with:74
91232023-12-10T18:53:29.267Z"msg":"[0]R flush_numbers[0..12]: [125, 116, 113, 119, 122, 122, 122, 128, 104, 127, 122, 122]"Wait for a query_work_queue command to finish before sending IO
91242023-12-10T18:53:29.267Z,"v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.053988917Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91252023-12-10T18:53:29.268Z{"msg":"[0]R generation[0..12]: [72, 67, 65, 68, 70, 70, 70, 73, 62, 72, 70, 70]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.054039906Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91262023-12-10T18:53:29.274ZError: Difference in extent metadata found!
91272023-12-10T18:53:29.274Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.054089263Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91282023-12-10T18:53:29.274Z{"msg":"[1]R flush_numbers[0..12]: [125, 116, 113, 119, 122, 122, 122, 128, 104, 127, 122, 122]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.054121454Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91292023-12-10T18:53:29.277Z{"msg":"[1]R generation[0..12]: [72, 67, 65, 68, 70, 70, 70, 73, 62, 72, 70, 70]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.054160389Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91302023-12-10T18:53:29.277Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.054200079Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91312023-12-10T18:53:29.277Z{"msg":"[2]R flush_numbers[0..12]: [125, 116, 113, 119, 122, 122, 122, 128, 104, 127, 122, 122]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.054241923Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91322023-12-10T18:53:29.277Z{"msg":"[2]R generation[0..12]: [72, 67, 65, 68, 70, 70, 70, 73, 62, 72, 70, 70]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.054282416Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91332023-12-10T18:53:29.277Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.054314332Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91342023-12-10T18:53:29.277Z{"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.054335758Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91352023-12-10T18:53:29.277Z{"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.054355814Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91362023-12-10T18:53:29.277Z{"msg":"Next flush: 129","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.054375212Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91372023-12-10T18:53:29.277Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.054394601Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91382023-12-10T18:53:29.278Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.054414498Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91392023-12-10T18:53:29.278Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.05443425Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91402023-12-10T18:53:29.278Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.054454521Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91412023-12-10T18:53:29.278Z{"msg":"5cf19ad2-8846-48a0-80f5-34d72e8cb959 is now active with session: 9804a389-432e-4b33-960f-0bda85c95ad6","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.054476253Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91422023-12-10T18:53:29.278Z{"msg":"5cf19ad2-8846-48a0-80f5-34d72e8cb959 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.054497203Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91432023-12-10T18:53:29.278Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.054517412Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91442023-12-10T18:53:29.278Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.054536865Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91452023-12-10T18:53:29.278Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.054575693Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91462023-12-10T18:53:29.278Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.054597672Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91472023-12-10T18:53:29.279Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.054617322Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91482023-12-10T18:53:29.279Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.054638863Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91492023-12-10T18:53:29.279Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.054658811Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91502023-12-10T18:53:29.279Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.0546784Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91512023-12-10T18:53:29.279Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.054706716Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91522023-12-10T18:53:29.279Z{Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
91532023-12-10T18:53:29.279Z"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.054779424Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91542023-12-10T18:53:29.280Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:29.054806697Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1458,"session_id":"9804a389-432e-4b33-960f-0bda85c95ad6"}
91552023-12-10T18:53:29.280ZLoading write count information from file "/tmp/test_repair_verify.data"
91562023-12-10T18:53:29.280ZRun Repair workload
91572023-12-10T18:53:29.280Z01/30 Write block 287 len 12288 data: 8 7 6
91582023-12-10T18:53:29.280Z02/30 Read block 9 len 12288
91592023-12-10T18:53:29.280Z03/30 Write block 226 len 36864 data: 8 7 6 6 6 6 6 5 9
91602023-12-10T18:53:29.280Z04/30 Flush
91612023-12-10T18:53:29.280Z05/30 Write block 578 len 20480 data: 6 6 6 7 6
91622023-12-10T18:53:29.280Z06/30 Write block 126 len 16384 data: 10 9 9 8
91632023-12-10T18:53:29.280Z07/30 Write block 280 len 4096 data: 5
91642023-12-10T18:53:29.280Z08/30 Read block 404 len 8192
91652023-12-10T18:53:29.281Z09/30 Read block 404 len 16384
91662023-12-10T18:53:29.281Z10/30 Read block 353 len 8192
91672023-12-10T18:53:29.281Z11/30 Read block 431 len 24576
91682023-12-10T18:53:29.281Z12/30 Read block 306 len 12288
91692023-12-10T18:53:29.281Z13/30 Write block 444 len 40960 data: 4 5 4 4 5 5 5 6 6 7
91702023-12-10T18:53:29.282Z14/30 Read block 225 len 20480
91712023-12-10T18:53:29.282Z15/30 Flush
91722023-12-10T18:53:29.282Z16/30 Write block 518 len 8192 data: 6 7
91732023-12-10T18:53:29.282Z17/30 Read block 146 len 8192
91742023-12-10T18:53:29.283Z18/30 Read block 37 len 24576
91752023-12-10T18:53:29.283Z19/30 Write block 248 len 4096 data: 8
91762023-12-10T18:53:29.283Z20/30 Read block 331 len 24576
91772023-12-10T18:53:29.283Z21/30 Write block 531 len 40960 data: 8 8 7 9 10 6 5 6 6 9
91782023-12-10T18:53:29.283Z22/30 Write block 559 len 12288 data: 4 4 6
91792023-12-10T18:53:29.283Z23/30 Read block 560 len 24576
91802023-12-10T18:53:29.283Z24/30 Write block 498 len 36864 data: 5 6 9 9 9 10 7 6 8
91812023-12-10T18:53:29.283Z25/30 Write block 30 len 24576 data: 5 6 8 5 5 6
91822023-12-10T18:53:29.283Z26/30 Read block 429 len 4096
91832023-12-10T18:53:29.284Z27/30 Write block 230 len 40960 data: 7 7 7 6 10 8 8 9 10 13
91842023-12-10T18:53:29.284Z28/30 Read block 193 len 4096
91852023-12-10T18:53:29.284Z29/30 Read block 255 len 36864
91862023-12-10T18:53:29.284Z30/30 Read block 190 len 16384
91872023-12-10T18:53:29.284Z----------------------------------------------------------------
91882023-12-10T18:53:29.284Z Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30
91892023-12-10T18:53:29.284ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
91902023-12-10T18:53:29.284Z 1 Acked 1000 Write 3 Done Done Done false
91912023-12-10T18:53:29.284Z 2 Acked 1001 Read 3 Done Sent Done false
91922023-12-10T18:53:29.284Z 3 Acked 1002 Write 9 Done Sent Done false
91932023-12-10T18:53:29.285Z 4 Acked 1003 Flush 0 Done Sent Done false
91942023-12-10T18:53:29.285Z 5 Acked 1004 Write 5 Done Sent Done false
91952023-12-10T18:53:29.285Z 6 Acked 1005 Write 4 Done Sent Done false
91962023-12-10T18:53:29.286Z 7 Acked 1006 Write 1 Done Sent Done false
91972023-12-10T18:53:29.286Z 8 Acked 1007 Read 2 Done Sent Done false
91982023-12-10T18:53:29.286Z 9 Acked 1008 Read 4 Done Sent Done false
91992023-12-10T18:53:29.286Z 10 Acked 1009 Read 2 Done Sent Done false
92002023-12-10T18:53:29.286Z 11 Acked 1010 Read 6 Done Sent Done false
92012023-12-10T18:53:29.286Z 12 Acked 1011 Read 3 Done Sent Done false
92022023-12-10T18:53:29.286Z 13 Acked 1012 Write 10 Done Sent Done false
92032023-12-10T18:53:29.286Z 14 Acked 1013 Read 5 Done Sent Done false
92042023-12-10T18:53:29.287Z 15 Acked 1014 Flush 0 Done Sent Done false
92052023-12-10T18:53:29.287Z 16 Acked 1015 Write 2 Done Sent Done false
92062023-12-10T18:53:29.287Z 17 Acked 1016 Read 2 Done Sent Done false
92072023-12-10T18:53:29.287Z 18 Acked 1017 Read 6 Done Sent Done false
92082023-12-10T18:53:29.287Z 19 Acked 1018 Write 1 Done Sent Done false
92092023-12-10T18:53:29.287Z 20 Acked 1019 Read 6 Done Sent Done false
92102023-12-10T18:53:29.287Z 21 Acked 1020 Write 10 Done Sent Done false
92112023-12-10T18:53:29.287Z 22 Acked 1021 Write 3 Done Sent Done false
92122023-12-10T18:53:29.287Z 23 Acked 1022 Read 6 Done Sent Done false
92132023-12-10T18:53:29.288Z 24 Acked 1023 Write 9 Done Sent Done false
92142023-12-10T18:53:29.288Z 25 Acked 1024 Write 6 Done Sent Done false
92152023-12-10T18:53:29.288Z 26 Acked 1025 Read 1 Done Sent Done false
92162023-12-10T18:53:29.288Z 27 Acked 1026 Write 10 Done Sent Done false
92172023-12-10T18:53:29.288Z 28 Acked 1027 Read 1 Done Sent Done false
92182023-12-10T18:53:29.289Z 29 Acked 1028 Read 9 Done Sent Done false
92192023-12-10T18:53:29.289Z 30 Acked 1029 Read 4 Done Sent Done false
92202023-12-10T18:53:29.289Z STATES DS:0 DS:1 DS:2 TOTAL
92212023-12-10T18:53:29.289Z New 0 0 0 0
92222023-12-10T18:53:29.289Z Sent 0 29 0 29
92232023-12-10T18:53:29.289Z Done 30 1 30 61
92242023-12-10T18:53:29.289Z Skipped 0 0 0 0
92252023-12-10T18:53:29.289Z Error 0 0 0 0
92262023-12-10T18:53:29.294ZLast Flush: 1014 0 1014
92272023-12-10T18:53:29.294ZDownstairs last five completed:
92282023-12-10T18:53:29.295ZUpstairs last five completed: 30 29 28 27 26 25
92292023-12-10T18:53:29.295ZWrote out file "/tmp/test_repair_verify.data"
92302023-12-10T18:53:29.295Z
92312023-12-10T18:53:29.295ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
92322023-12-10T18:53:29.295ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
92332023-12-10T18:53:29.295Z 0 000-019 72 72 72 125 125 125 F F F
92342023-12-10T18:53:29.295Z 1 020-039 67 67 67 116 116 116 T F T <---
92352023-12-10T18:53:29.296Z 2 040-059 65 65 65 113 113 113 F F F
92362023-12-10T18:53:29.296Z 3 060-079 68 68 68 119 119 119 F F F
92372023-12-10T18:53:29.296Z 4 080-099 70 70 70 122 122 122 F F F
92382023-12-10T18:53:29.296Z 5 100-119 70 70 70 122 122 122 F F F
92392023-12-10T18:53:29.296Z 6 120-139 74 70 74 130 122 130 F F F <---
92402023-12-10T18:53:29.296Z 7 140-159 73 73 73 128 128 128 F F F
92412023-12-10T18:53:29.297Z 8 160-179 62 62 62 104 104 104 F F F
92422023-12-10T18:53:29.297Z 9 180-199 72 72 72 127 127 127 F F F
92432023-12-10T18:53:29.298Z 10 200-219 70 70 70 122 122 122 F F F
92442023-12-10T18:53:29.298Z 11 220-239 74 70 74 129 122 129 T F T <---
92452023-12-10T18:53:29.298Z 12 240-259 69 69 69 121 121 121 T F T <---
92462023-12-10T18:53:29.298Z 13 260-279 73 73 73 128 128 128 F F F
92472023-12-10T18:53:29.299Z 14 280-299 74 73 74 130 128 130 F T F <---
92482023-12-10T18:53:29.299Z 15 300-319 70 70 70 122 122 122 F F F
92492023-12-10T18:53:29.299Z 16 320-339 64 64 64 110 110 110 F F F
92502023-12-10T18:53:29.299Z 17 340-359 70 70 70 122 122 122 F F F
92512023-12-10T18:53:29.299Z 18 360-379 73 73 73 128 128 128 F F F
92522023-12-10T18:53:29.299Z 19 380-399 68 68 68 117 117 117 F F F
92532023-12-10T18:53:29.300Z 20 400-419 72 72 72 127 127 127 F F F
92542023-12-10T18:53:29.300Z 21 420-439 59 59 59 98 98 98 F F F
92552023-12-10T18:53:29.300Z 22 440-459 74 70 74 130 122 130 F F F <---
92562023-12-10T18:53:29.304Z 23 460-479 72 72 72 125 125 125 F F F
92572023-12-10T18:53:29.304Z 24 480-499 72 72 72 126 126 126 T F T <---
92582023-12-10T18:53:29.304Z 25 500-519 72 72 72 126 126 126 T F T <---
92592023-12-10T18:53:29.304Z 26 520-539 64 64 64 107 107 107 T F T <---
92602023-12-10T18:53:29.304Z 27 540-559 73 73 73 128 128 128 T F T <---
92612023-12-10T18:53:29.304Z 28 560-579 74 68 74 130 120 130 T F T <---
92622023-12-10T18:53:29.305Z 29 580-599 74 73 74 130 128 130 F F F <---
92632023-12-10T18:53:29.305ZMax gen: 74, Max flush: 130
92642023-12-10T18:53:29.305ZA Difference in extent metadata is expected here
92652023-12-10T18:53:29.305ZOn loop 37
92662023-12-10T18:53:29.305Z
92672023-12-10T18:53:29.305ZVerifying data now
92682023-12-10T18:53:34.359ZNo change after verify
92692023-12-10T18:53:34.359ZLoop: 37 Downstairs dump after verify (and repair):
92702023-12-10T18:53:34.381ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
92712023-12-10T18:53:34.381Z 0 000-019 72 72 72 125 125 125 F F F
92722023-12-10T18:53:34.398Z 1 020-039 75 75 75 131 131 131 F F F
92732023-12-10T18:53:34.398Z 2 040-059 65 65 65 113 113 113 F F F
92742023-12-10T18:53:34.398Z 3 060-079 68 68 68 119 119 119 F F F
92752023-12-10T18:53:34.399Z 4 080-099 70 70 70 122 122 122 F F F
92762023-12-10T18:53:34.399Z 5 100-119 70 70 70 122 122 122 F F F
92772023-12-10T18:53:34.399Z 6 120-139 74 74 74 130 130 130 F F F
92782023-12-10T18:53:34.399Z 7 140-159 73 73 73 128 128 128 F F F
92792023-12-10T18:53:34.399Z 8 160-179 62 62 62 104 104 104 F F F
92802023-12-10T18:53:34.399Z 9 180-199 72 72 72 127 127 127 F F F
92812023-12-10T18:53:34.400Z 10 200-219 70 70 70 122 122 122 F F F
92822023-12-10T18:53:34.401Z 11 220-239 75 75 75 131 131 131 F F F
92832023-12-10T18:53:34.401Z 12 240-259 75 75 75 131 131 131 F F F
92842023-12-10T18:53:34.401Z 13 260-279 73 73 73 128 128 128 F F F
92852023-12-10T18:53:34.402Z 14 280-299 74 74 74 130 130 130 F F F
92862023-12-10T18:53:34.402Z 15 300-319 70 70 70 122 122 122 F F F
92872023-12-10T18:53:34.402Z 16 320-339 64 64 64 110 110 110 F F F
92882023-12-10T18:53:34.403Z 17 340-359 70 70 70 122 122 122 F F F
92892023-12-10T18:53:34.403Z 18 360-379 73 73 73 128 128 128 F F F
92902023-12-10T18:53:34.403Z 19 380-399 68 68 68 117 117 117 F F F
92912023-12-10T18:53:34.404Z 20 400-419 72 72 72 127 127 127 F F F
92922023-12-10T18:53:34.404Z 21 420-439 59 59 59 98 98 98 F F F
92932023-12-10T18:53:34.404Z 22 440-459 74 74 74 130 130 130 F F F
92942023-12-10T18:53:34.404Z 23 460-479 72 72 72 125 125 125 F F F
92952023-12-10T18:53:34.404Z 24 480-499 75 75 75 131 131 131 F F F
92962023-12-10T18:53:34.405Z 25 500-519 75 75 75 131 131 131 F F F
92972023-12-10T18:53:34.405Z 26 520-539 75 75 75 131 131 131 F F F
92982023-12-10T18:53:34.405Z 27 540-559 75 75 75 131 131 131 F F F
92992023-12-10T18:53:34.406Z 28 560-579 75 75 75 131 131 131 F F F
93002023-12-10T18:53:34.406Z 29 580-599 74 74 74 130 130 130 F F F
93012023-12-10T18:53:34.406ZMax gen: 75, Max flush: 131
93022023-12-10T18:53:34.407Z
93032023-12-10T18:53:34.407ZBegin loop 38
93042023-12-10T18:53:34.407ZDownstairs to restart: 2
93052023-12-10T18:53:34.421Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.415957589Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466}
93062023-12-10T18:53:34.421Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.416361123Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466}
93072023-12-10T18:53:34.422Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.416494918Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466}
93082023-12-10T18:53:34.422Z{"msg":"Crucible 4d2b02d4-b8e6-4a0d-8a90-d536f5582268 has session id: a4e39fc7-5e5f-409b-abe6-5994d3b25a8f","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.416537118Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466Crucible runtime is spawned
93092023-12-10T18:53:34.423Z,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93102023-12-10T18:53:34.423Z{"msg":"Upstairs opts: Upstairs UUID: 4d2b02d4-b8e6-4a0d-8a90-d536f5582268, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.416595162Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93112023-12-10T18:53:34.423Z{"msg":"Crucible stats registered with UUID: 4d2b02d4-b8e6-4a0d-8a90-d536f5582268","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.41663369Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}The guest has requested activation with gen:76
93122023-12-10T18:53:34.424Z
93132023-12-10T18:53:34.424Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.416779707Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f","task":"up_listen"}
93142023-12-10T18:53:34.424Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.416836539Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93152023-12-10T18:53:34.424Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.416929878Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93162023-12-10T18:53:34.424Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.416971771Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"looper":"0","session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93172023-12-10T18:53:34.450Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.417238668Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"looper":"1","session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93182023-12-10T18:53:34.451Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.417300539Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"looper":"2","session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93192023-12-10T18:53:34.451Z{"msg":"4d2b02d4-b8e6-4a0d-8a90-d536f5582268 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.417326578Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93202023-12-10T18:53:34.452Z{"msg":"Set desired generation to :76","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.417381835Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93212023-12-10T18:53:34.452Z{"msg":"[2] 4d2b02d4-b8e6-4a0d-8a90-d536f5582268 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.417427241Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"looper":"2","session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93222023-12-10T18:53:34.452Z{"msg":"[1] 4d2b02d4-b8e6-4a0d-8a90-d536f5582268 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.417469287Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"looper":"1","session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93232023-12-10T18:53:34.452Z{"msg":"[0] 4d2b02d4-b8e6-4a0d-8a90-d536f5582268 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.417536166Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"looper":"0","session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93242023-12-10T18:53:34.453Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.417579787Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93252023-12-10T18:53:34.453Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.417623527Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93262023-12-10T18:53:34.453Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.417760174Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93272023-12-10T18:53:34.454Z{"msg":"[0] 4d2b02d4-b8e6-4a0d-8a90-d536f5582268 (a4e39fc7-5e5f-409b-abe6-5994d3b25a8f) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.418167704Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93282023-12-10T18:53:34.455Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.418282179Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93292023-12-10T18:53:34.455Z{"msg":"[0] client is_active_req TRUE, promote! session a4e39fc7-5e5f-409b-abe6-5994d3b25a8f","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.418371006Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93302023-12-10T18:53:34.455Z{"msg":"[1] 4d2b02d4-b8e6-4a0d-8a90-d536f5582268 (a4e39fc7-5e5f-409b-abe6-5994d3b25a8f) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.418461112Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93312023-12-10T18:53:34.455Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.418501448Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93322023-12-10T18:53:34.455Z{"msg":"[1] client is_active_req TRUE, promote! session a4e39fc7-5e5f-409b-abe6-5994d3b25a8f","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.418666397Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93332023-12-10T18:53:34.456Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.419009223Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93342023-12-10T18:53:34.456Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.419089582Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93352023-12-10T18:53:34.456Z{"msg":"4d2b02d4-b8e6-4a0d-8a90-d536f5582268 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.419159213Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93362023-12-10T18:53:34.457Z{"msg":"[2] 4d2b02d4-b8e6-4a0d-8a90-d536f5582268 (a4e39fc7-5e5f-409b-abe6-5994d3b25a8f) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.41921837Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93372023-12-10T18:53:34.458Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.419257621Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93382023-12-10T18:53:34.458Z{"msg":"[2] client is_active_req TRUE, promote! session a4e39fc7-5e5f-409b-abe6-5994d3b25a8f","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.419294843Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93392023-12-10T18:53:34.458Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.419371023Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93402023-12-10T18:53:34.458Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.419422823Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93412023-12-10T18:53:34.458Z{"msg":"4d2b02d4-b8e6-4a0d-8a90-d536f5582268 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.419461658Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93422023-12-10T18:53:34.459Z{"msg":"[0] 4d2b02d4-b8e6-4a0d-8a90-d536f5582268 (a4e39fc7-5e5f-409b-abe6-5994d3b25a8f) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.419985021Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93432023-12-10T18:53:34.459Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.420089263Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93442023-12-10T18:53:34.459Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:34.420129383Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93452023-12-10T18:53:34.459Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.420220762Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93462023-12-10T18:53:34.459Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.420304109Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93472023-12-10T18:53:34.459Z{"msg":"4d2b02d4-b8e6-4a0d-8a90-d536f5582268 WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.420380813Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93482023-12-10T18:53:34.459Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.420449479Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93492023-12-10T18:53:34.460Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.420476126Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93502023-12-10T18:53:34.460Z{"msg":"[1] 4d2b02d4-b8e6-4a0d-8a90-d536f5582268 (a4e39fc7-5e5f-409b-abe6-5994d3b25a8f) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.420589816Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93512023-12-10T18:53:34.460Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.420647457Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93522023-12-10T18:53:34.460Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:34.420670365Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93532023-12-10T18:53:34.460Z{"msg":"4d2b02d4-b8e6-4a0d-8a90-d536f5582268 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.420691091Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93542023-12-10T18:53:34.460Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.42077417Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93552023-12-10T18:53:34.460Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.420813567Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93562023-12-10T18:53:34.461Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.420871325Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93572023-12-10T18:53:34.461Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.420911324Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93582023-12-10T18:53:34.461Z{"msg":"4d2b02d4-b8e6-4a0d-8a90-d536f5582268 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.420968483Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93592023-12-10T18:53:34.461Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.421008252Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93602023-12-10T18:53:34.461Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.421061703Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93612023-12-10T18:53:34.461Z{"msg":"[2] 4d2b02d4-b8e6-4a0d-8a90-d536f5582268 (a4e39fc7-5e5f-409b-abe6-5994d3b25a8f) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.421105091Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93622023-12-10T18:53:34.461Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.421159307Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93632023-12-10T18:53:34.461Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:34.421197897Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93642023-12-10T18:53:34.461ZThe guest has finished waiting for activation with:76
93652023-12-10T18:53:34.461Z{"msg":"Wait for a query_work_queue command to finish before sending IO
93662023-12-10T18:53:34.462Z[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.421256148Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93672023-12-10T18:53:34.462Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.421295201Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93682023-12-10T18:53:34.462Z{"msg":"4d2b02d4-b8e6-4a0d-8a90-d536f5582268 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.421353852Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93692023-12-10T18:53:34.462Z{"msg":"[0]R flush_numbers[0..12]: [125, 131, 113, 119, 122, 122, 130, 128, 104, 127, 122, 131]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.421395862Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93702023-12-10T18:53:34.462Z{"msg":"[0]R generation[0..12]: [72, 75, 65, 68, 70, 70, 74, 73, 62, 72, 70, 75]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.42146109Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93712023-12-10T18:53:34.462Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.421581686Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93722023-12-10T18:53:34.462Z{"msg":"[1]R flush_numbers[0..12]: [125, 131, 113, 119, 122, 122, 130, 128, 104, 127, 122, 131]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.421625379Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93732023-12-10T18:53:34.462Z{"msg":"[1]R generation[0..12]: [72, 75, 65, 68, 70, 70, 74, 73, 62, 72, 70, 75]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.421685769Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93742023-12-10T18:53:34.462Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.421727829Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93752023-12-10T18:53:34.462Z{"msg":"[2]R flush_numbers[0..12]: [125, 131, 113, 119, 122, 122, 130, 128, 104, 127, 122, 131]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.421783237Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93762023-12-10T18:53:34.462Z{"msg":"[2]R generation[0..12]: [72, 75, 65, 68, 70, 70, 74, 73, 62, 72, 70, 75]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.421822988Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93772023-12-10T18:53:34.463Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.421886045Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93782023-12-10T18:53:34.463Z{"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.421924085Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93792023-12-10T18:53:34.463Z{"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.421980556Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93802023-12-10T18:53:34.463Z{"msg":"Next flush: 132","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.422019016Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93812023-12-10T18:53:34.463ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
93822023-12-10T18:53:34.463Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.422090466Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93832023-12-10T18:53:34.463Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.422128429Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93842023-12-10T18:53:34.463Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.422186303Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93852023-12-10T18:53:34.463Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.42222553Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93862023-12-10T18:53:34.463Z{"msg":"4d2b02d4-b8e6-4a0d-8a90-d536f5582268 is now active with session: a4e39fc7-5e5f-409b-abe6-5994d3b25a8f","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.422281497Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93872023-12-10T18:53:34.464Z{"msg":"4d2b02d4-b8e6-4a0d-8a90-d536f5582268 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.422321739Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93882023-12-10T18:53:34.464Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.422374915Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93892023-12-10T18:53:34.464Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.422412805Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93902023-12-10T18:53:34.464Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.422466399Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93912023-12-10T18:53:34.464Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.422504766Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93922023-12-10T18:53:34.464Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.422574972Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93932023-12-10T18:53:34.464Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible",Loading write count information from file "/tmp/test_repair_verify.data"
93942023-12-10T18:53:34.464Z"level":30,"time":"2023-12-10T18:53:34.422621726Z","hostname":"Run Repair workload
93952023-12-10T18:53:34.464Zip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93962023-12-10T18:53:34.464Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.422721995Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93972023-12-10T18:53:34.465Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.422776435Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93982023-12-10T18:53:34.465Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.422815277Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
93992023-12-10T18:53:34.465Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.422869353Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
94002023-12-10T18:53:34.465Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:34.422907134Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1466,"session_id":"a4e39fc7-5e5f-409b-abe6-5994d3b25a8f"}
94012023-12-10T18:53:34.465Z01/30 Write block 375 len 16384 data: 9 9 8 11
94022023-12-10T18:53:34.465Z02/30 Write block 51 len 16384 data: 6 5 4 5
94032023-12-10T18:53:34.465Z03/30 Read block 32 len 4096
94042023-12-10T18:53:34.465Z04/30 Read block 132 len 24576
94052023-12-10T18:53:34.465Z05/30 Write block 348 len 12288 data: 6 7 6
94062023-12-10T18:53:34.465Z06/30 Read block 300 len 28672
94072023-12-10T18:53:34.465Z07/30 Flush
94082023-12-10T18:53:34.465Z08/30 Flush
94092023-12-10T18:53:34.479Z09/30 Write block 269 len 40960 data: 6 8 7 7 7 8 9 8 8 9
94102023-12-10T18:53:34.479Z10/30 Read block 392 len 16384
94112023-12-10T18:53:34.479Z11/30 Read block 34 len 8192
94122023-12-10T18:53:34.479Z12/30 Read block 22 len 24576
94132023-12-10T18:53:34.479Z13/30 Read block 125 len 16384
94142023-12-10T18:53:34.480Z14/30 Read block 115 len 24576
94152023-12-10T18:53:34.480Z15/30 Read block 482 len 24576
94162023-12-10T18:53:34.480Z16/30 Write block 26 len 12288 data: 7 5 5
94172023-12-10T18:53:34.480Z17/30 Read block 460 len 8192
94182023-12-10T18:53:34.480Z18/30 Write block 63 len 16384 data: 3 5 5 6
94192023-12-10T18:53:34.480Z19/30 Read block 341 len 24576
94202023-12-10T18:53:34.480Z20/30 Write block 103 len 4096 data: 7
94212023-12-10T18:53:34.480Z21/30 Write block 93 len 40960 data: 5 5 4 3 3 5 6 7 6 8
94222023-12-10T18:53:34.481Z22/30 Write block 151 len 16384 data: 5 5 4 6
94232023-12-10T18:53:34.481Z23/30 Write block 203 len 16384 data: 8 7 7 5
94242023-12-10T18:53:34.481Z24/30 Write block 531 len 20480 data: 9 9 8 10 11
94252023-12-10T18:53:34.481Z25/30 Write block 196 len 8192 data: 10 11
94262023-12-10T18:53:34.481Z26/30 Read block 295 len 32768
94272023-12-10T18:53:34.481Z27/30 Write block 321 len 20480 data: 7 9 8 9 7
94282023-12-10T18:53:34.481Z28/30 Write block 108 len 40960 data: 7 6 6 5 5 6 4 6 8 9
94292023-12-10T18:53:34.481Z29/30 Write block 20 len 16384 data: 6 5 5 5
94302023-12-10T18:53:34.481Z30/30 Write block 388 len 12288 data: 8 7 7
94312023-12-10T18:53:34.482Z----------------------------------------------------------------
94322023-12-10T18:53:34.482Z Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:30
94332023-12-10T18:53:34.482ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
94342023-12-10T18:53:34.482Z 1 Acked 1000 Write 4 Done Done Sent false
94352023-12-10T18:53:34.482Z 2 Acked 1001 Write 4 Done Done Sent false
94362023-12-10T18:53:34.482Z 3 Acked 1002 Read 1 Done Done Sent false
94372023-12-10T18:53:34.482Z 4 Acked 1003 Read 6 Done Done Sent false
94382023-12-10T18:53:34.482Z 5 Acked 1004 Write 3 Done Done Sent false
94392023-12-10T18:53:34.482Z 6 Acked 1005 Read 7 Done Done Sent false
94402023-12-10T18:53:34.482Z 7 Acked 1006 Flush 0 Done Done Sent false
94412023-12-10T18:53:34.482Z 8 Acked 1007 Flush 0 Done Done Sent false
94422023-12-10T18:53:34.483Z 9 Acked 1008 Write 10 Done Done Sent false
94432023-12-10T18:53:34.483Z 10 Acked 1009 Read 4 Done Done Sent false
94442023-12-10T18:53:34.483Z 11 Acked 1010 Read 2 Done Done Sent false
94452023-12-10T18:53:34.483Z 12 Acked 1011 Read 6 Done Done Sent false
94462023-12-10T18:53:34.483Z 13 Acked 1012 Read 4 Done Done Sent false
94472023-12-10T18:53:34.483Z 14 Acked 1013 Read 6 Done Done Sent false
94482023-12-10T18:53:34.483Z 15 Acked 1014 Read 6 Done Done Sent false
94492023-12-10T18:53:34.485Z 16 Acked 1015 Write 3 Done Done Sent false
94502023-12-10T18:53:34.485Z 17 Acked 1016 Read 2 Done Done Sent false
94512023-12-10T18:53:34.485Z 18 Acked 1017 Write 4 Done Done Sent false
94522023-12-10T18:53:34.485Z 19 Acked 1018 Read 6 Done Done Sent false
94532023-12-10T18:53:34.486Z 20 Acked 1019 Write 1 Done Done Sent false
94542023-12-10T18:53:34.486Z 21 Acked 1020 Write 10 Done Done Sent false
94552023-12-10T18:53:34.486Z 22 Acked 1021 Write 4 Done Done Sent false
94562023-12-10T18:53:34.486Z 23 Acked 1022 Write 4 Done Done Sent false
94572023-12-10T18:53:34.486Z 24 Acked 1023 Write 5 Done Done Sent false
94582023-12-10T18:53:34.486Z 25 Acked 1024 Write 2 Done Done Sent false
94592023-12-10T18:53:34.486Z 26 Acked 1025 Read 8 Done Done Sent false
94602023-12-10T18:53:34.487Z 27 Acked 1026 Write 5 Done Done Sent false
94612023-12-10T18:53:34.487Z 28 Acked 1027 Write 10 Done Done Sent false
94622023-12-10T18:53:34.487Z 29 Acked 1028 Write 4 Done Done Sent false
94632023-12-10T18:53:34.487Z 30 Acked 1029 Write 3 Sent Sent Sent false
94642023-12-10T18:53:34.487Z STATES DS:0 DS:1 DS:2 TOTAL
94652023-12-10T18:53:34.487Z New 0 0 0 0
94662023-12-10T18:53:34.487Z Sent 1 1 30 32
94672023-12-10T18:53:34.487Z Done 29 29 0 58
94682023-12-10T18:53:34.487Z Skipped 0 0 0 0
94692023-12-10T18:53:34.487Z Error 0 0 0 0
94702023-12-10T18:53:34.487ZLast Flush: 1007 1007 0
94712023-12-10T18:53:34.487ZDownstairs last five completed:
94722023-12-10T18:53:34.487ZUpstairs last five completed: 30 29 28 27 26 25
94732023-12-10T18:53:34.488ZWrote out file "/tmp/test_repair_verify.data"
94742023-12-10T18:53:34.498Z
94752023-12-10T18:53:34.498ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
94762023-12-10T18:53:34.518ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
94772023-12-10T18:53:34.518Z 0 000-019 72 72 72 125 125 125 F F F
94782023-12-10T18:53:34.518Z 1 020-039 75 75 75 131 131 131 T T F <---
94792023-12-10T18:53:34.518Z 2 040-059 76 76 65 132 132 113 F F F <---
94802023-12-10T18:53:34.518Z 3 060-079 68 68 68 119 119 119 T T F <---
94812023-12-10T18:53:34.518Z 4 080-099 70 70 70 122 122 122 T T F <---
94822023-12-10T18:53:34.519Z 5 100-119 70 70 70 122 122 122 T T F <---
94832023-12-10T18:53:34.519Z 6 120-139 74 74 74 130 130 130 F F F
94842023-12-10T18:53:34.519Z 7 140-159 73 73 73 128 128 128 T T F <---
94852023-12-10T18:53:34.519Z 8 160-179 62 62 62 104 104 104 F F F
94862023-12-10T18:53:34.519Z 9 180-199 72 72 72 127 127 127 T T F <---
94872023-12-10T18:53:34.519Z 10 200-219 70 70 70 122 122 122 T T F <---
94882023-12-10T18:53:34.519Z 11 220-239 75 75 75 131 131 131 F F F
94892023-12-10T18:53:34.519Z 12 240-259 75 75 75 131 131 131 F F F
94902023-12-10T18:53:34.520Z 13 260-279 73 73 73 128 128 128 T T F <---
94912023-12-10T18:53:34.520Z 14 280-299 74 74 74 130 130 130 F F F
94922023-12-10T18:53:34.520Z 15 300-319 70 70 70 122 122 122 F F F
94932023-12-10T18:53:34.520Z 16 320-339 64 64 64 110 110 110 T T F <---
94942023-12-10T18:53:34.520Z 17 340-359 76 76 70 132 132 122 F F F <---
94952023-12-10T18:53:34.520Z 18 360-379 76 76 73 132 132 128 F F F <---
94962023-12-10T18:53:34.520Z 19 380-399 68 68 68 117 117 117 T T F <---
94972023-12-10T18:53:34.521Z 20 400-419 72 72 72 127 127 127 F F F
94982023-12-10T18:53:34.521Z 21 420-439 59 59 59 98 98 98 F F F
94992023-12-10T18:53:34.521Z 22 440-459 74 74 74 130 130 130 F F F
95002023-12-10T18:53:34.521Z 23 460-479 72 72 72 125 125 125 F F F
95012023-12-10T18:53:34.521Z 24 480-499 75 75 75 131 131 131 F F F
95022023-12-10T18:53:34.521Z 25 500-519 75 75 75 131 131 131 F F F
95032023-12-10T18:53:34.522Z 26 520-539 75 75 75 131 131 131 T T F <---
95042023-12-10T18:53:34.522Z 27 540-559 75 75 75 131 131 131 F F F
95052023-12-10T18:53:34.522Z 28 560-579 75 75 75 131 131 131 F F F
95062023-12-10T18:53:34.522Z 29 580-599 74 74 74 130 130 130 F F F
95072023-12-10T18:53:34.522ZMax gen: 76, Max flush: 132
95082023-12-10T18:53:34.522ZError: Difference in extent metadata found!
95092023-12-10T18:53:34.546ZA Difference in extent metadata is expected here
95102023-12-10T18:53:34.547ZOn loop 38
95112023-12-10T18:53:34.547Z
95122023-12-10T18:53:34.547ZVerifying data now
95132023-12-10T18:53:39.878ZNo change after verify
95142023-12-10T18:53:39.878ZLoop: 38 Downstairs dump after verify (and repair):
95152023-12-10T18:53:39.913ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
95162023-12-10T18:53:39.913Z 0 000-019 72 72 72 125 125 125 F F F
95172023-12-10T18:53:39.936Z 1 020-039 77 77 77 133 133 133 F F F
95182023-12-10T18:53:39.936Z 2 040-059 76 76 76 132 132 132 F F F
95192023-12-10T18:53:39.936Z 3 060-079 77 77 77 133 133 133 F F F
95202023-12-10T18:53:39.936Z 4 080-099 77 77 77 133 133 133 F F F
95212023-12-10T18:53:39.936Z 5 100-119 77 77 77 133 133 133 F F F
95222023-12-10T18:53:39.937Z 6 120-139 74 74 74 130 130 130 F F F
95232023-12-10T18:53:39.937Z 7 140-159 77 77 77 133 133 133 F F F
95242023-12-10T18:53:39.937Z 8 160-179 62 62 62 104 104 104 F F F
95252023-12-10T18:53:39.937Z 9 180-199 77 77 77 133 133 133 F F F
95262023-12-10T18:53:39.937Z 10 200-219 77 77 77 133 133 133 F F F
95272023-12-10T18:53:39.937Z 11 220-239 75 75 75 131 131 131 F F F
95282023-12-10T18:53:39.937Z 12 240-259 75 75 75 131 131 131 F F F
95292023-12-10T18:53:39.937Z 13 260-279 77 77 77 133 133 133 F F F
95302023-12-10T18:53:39.937Z 14 280-299 74 74 74 130 130 130 F F F
95312023-12-10T18:53:39.937Z 15 300-319 70 70 70 122 122 122 F F F
95322023-12-10T18:53:39.937Z 16 320-339 77 77 77 133 133 133 F F F
95332023-12-10T18:53:39.937Z 17 340-359 76 76 76 132 132 132 F F F
95342023-12-10T18:53:39.938Z 18 360-379 76 76 76 132 132 132 F F F
95352023-12-10T18:53:39.938Z 19 380-399 77 77 77 133 133 133 F F F
95362023-12-10T18:53:39.939Z 20 400-419 72 72 72 127 127 127 F F F
95372023-12-10T18:53:39.939Z 21 420-439 59 59 59 98 98 98 F F F
95382023-12-10T18:53:39.939Z 22 440-459 74 74 74 130 130 130 F F F
95392023-12-10T18:53:39.939Z 23 460-479 72 72 72 125 125 125 F F F
95402023-12-10T18:53:39.939Z 24 480-499 75 75 75 131 131 131 F F F
95412023-12-10T18:53:39.939Z 25 500-519 75 75 75 131 131 131 F F F
95422023-12-10T18:53:39.939Z 26 520-539 77 77 77 133 133 133 F F F
95432023-12-10T18:53:39.940Z 27 540-559 75 75 75 131 131 131 F F F
95442023-12-10T18:53:39.940Z 28 560-579 75 75 75 131 131 131 F F F
95452023-12-10T18:53:39.940Z 29 580-599 74 74 74 130 130 130 F F F
95462023-12-10T18:53:39.940ZMax gen: 77, Max flush: 133
95472023-12-10T18:53:39.940Z
95482023-12-10T18:53:39.940ZBegin loop 39
95492023-12-10T18:53:39.941ZDownstairs to restart: 0
95502023-12-10T18:53:40.366Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.936734641Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474}
95512023-12-10T18:53:40.366Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.937130843Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474}
95522023-12-10T18:53:40.366Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.937185573Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474}
95532023-12-10T18:53:40.368Z{"msg":"Crucible d8b84b20-8ce9-4e5b-913e-46c8eaa92421 has session id: 29f78388-2fa0-40e3-a5fa-209c0d081f1a","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.937209532Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95542023-12-10T18:53:40.368Z{"msg":"Upstairs opts: Upstairs UUID: d8b84b20-8ce9-4e5b-913e-46c8eaa92421, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.937275207Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95552023-12-10T18:53:40.369Z{"msg":"Crucible stats registered with UUID: d8b84b20-8ce9-4e5b-913e-46c8eaa92421","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.937348912Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95562023-12-10T18:53:40.369ZCrucible runtime is spawned
95572023-12-10T18:53:40.369ZThe guest has requested activation with gen:78
95582023-12-10T18:53:40.369Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.937640092Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a","task":"up_listen"}
95592023-12-10T18:53:40.369Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.937707352Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95602023-12-10T18:53:40.369Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.937733642Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"looper":"0","session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95612023-12-10T18:53:40.369Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.937763027Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95622023-12-10T18:53:40.369Z{"msg":"[0] d8b84b20-8ce9-4e5b-913e-46c8eaa92421 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.93791748Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"looper":"0","session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95632023-12-10T18:53:40.369Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.937986786Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"looper":"1","session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95642023-12-10T18:53:40.371Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.938288599Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"looper":"2","session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95652023-12-10T18:53:40.371Z{"msg":"[1] d8b84b20-8ce9-4e5b-913e-46c8eaa92421 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.938371381Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"looper":"1","session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95662023-12-10T18:53:40.371Z{"msg":"[2] d8b84b20-8ce9-4e5b-913e-46c8eaa92421 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.938520498Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"looper":"2","session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95672023-12-10T18:53:40.371Z{"msg":"d8b84b20-8ce9-4e5b-913e-46c8eaa92421 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.938921816Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95682023-12-10T18:53:40.371Z{"msg":"Set desired generation to :78","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.939006673Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95692023-12-10T18:53:40.371Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.939066159Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95702023-12-10T18:53:40.372Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.939373333Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95712023-12-10T18:53:40.372Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.939460712Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95722023-12-10T18:53:40.372Z{"msg":"[1] d8b84b20-8ce9-4e5b-913e-46c8eaa92421 (29f78388-2fa0-40e3-a5fa-209c0d081f1a) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.939867258Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95732023-12-10T18:53:40.372Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.939985005Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95742023-12-10T18:53:40.373Z{"msg":"[1] client is_active_req TRUE, promote! session 29f78388-2fa0-40e3-a5fa-209c0d081f1a","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.940074493Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95752023-12-10T18:53:40.373Z{"msg":"[2] d8b84b20-8ce9-4e5b-913e-46c8eaa92421 (29f78388-2fa0-40e3-a5fa-209c0d081f1a) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.940152686Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95762023-12-10T18:53:40.373Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.940182554Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95772023-12-10T18:53:40.374Z{"msg":"[2] client is_active_req TRUE, promote! session 29f78388-2fa0-40e3-a5fa-209c0d081f1a","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.940232037Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95782023-12-10T18:53:40.374Z{"msg":"[0] d8b84b20-8ce9-4e5b-913e-46c8eaa92421 (29f78388-2fa0-40e3-a5fa-209c0d081f1a) New WaitActive WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.9403547Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95792023-12-10T18:53:40.374Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.940423386Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95802023-12-10T18:53:40.374Z{"msg":"[0] client is_active_req TRUE, promote! session 29f78388-2fa0-40e3-a5fa-209c0d081f1a","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.940465868Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95812023-12-10T18:53:40.374Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.940581302Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95822023-12-10T18:53:40.374Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.940678733Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95832023-12-10T18:53:40.375Z{"msg":"d8b84b20-8ce9-4e5b-913e-46c8eaa92421 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.940772541Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95842023-12-10T18:53:40.376Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.940867011Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95852023-12-10T18:53:40.376Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.940914964Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95862023-12-10T18:53:40.376Z{"msg":"d8b84b20-8ce9-4e5b-913e-46c8eaa92421 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.940948242Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95872023-12-10T18:53:40.376Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.941202928Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95882023-12-10T18:53:40.376Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.941290638Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95892023-12-10T18:53:40.376Z{"msg":"d8b84b20-8ce9-4e5b-913e-46c8eaa92421 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.941391959Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95902023-12-10T18:53:40.377Z{"msg":"[1] d8b84b20-8ce9-4e5b-913e-46c8eaa92421 (29f78388-2fa0-40e3-a5fa-209c0d081f1a) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.941538694Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95912023-12-10T18:53:40.377Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.941615112Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95922023-12-10T18:53:40.377Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:39.941657214Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95932023-12-10T18:53:40.377Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.941727305Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95942023-12-10T18:53:40.377Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.941861688Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95952023-12-10T18:53:40.377Z{"msg":"d8b84b20-8ce9-4e5b-913e-46c8eaa92421 WaitActive WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.941931214Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95962023-12-10T18:53:40.377Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.942000378Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95972023-12-10T18:53:40.378Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.942040467Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95982023-12-10T18:53:40.378Z{"msg":"[2] d8b84b20-8ce9-4e5b-913e-46c8eaa92421 (29f78388-2fa0-40e3-a5fa-209c0d081f1a) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.942117017Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
95992023-12-10T18:53:40.378Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.942243011Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96002023-12-10T18:53:40.378Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:39.942283315Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96012023-12-10T18:53:40.379Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.942337303Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96022023-12-10T18:53:40.379Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:78
96032023-12-10T18:53:40.380ZWait for a query_work_queue command to finish before sending IO
96042023-12-10T18:53:40.380Z,"time":"2023-12-10T18:53:39.942375145Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96052023-12-10T18:53:40.380Z{"msg":"d8b84b20-8ce9-4e5b-913e-46c8eaa92421 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.942525624Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96062023-12-10T18:53:40.380Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.942613505Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96072023-12-10T18:53:40.381Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.942686941Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96082023-12-10T18:53:40.381Z{"msg":"[0] d8b84b20-8ce9-4e5b-913e-46c8eaa92421 (29f78388-2fa0-40e3-a5fa-209c0d081f1a) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.94273141Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96092023-12-10T18:53:40.381Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.94276697Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96102023-12-10T18:53:40.382Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:39.942842069Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96112023-12-10T18:53:40.382Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.94291615Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96122023-12-10T18:53:40.382Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.94298542Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96132023-12-10T18:53:40.382Z{"msg":"d8b84b20-8ce9-4e5b-913e-46c8eaa92421 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.943055671Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96142023-12-10T18:53:40.383Z{"msg":"[0]R flush_numbers[0..12]: [125, 133, 132, 133, 133, 133, 130, 133, 104, 133, 133, 131]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.943128271Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96152023-12-10T18:53:40.384Z{"msg":"[0]R generation[0..12]: [72, 77, 76, 77, 77, 77, 74, 77, 62, 77, 77, 75]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.943172251Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96162023-12-10T18:53:40.384Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.943242268Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96172023-12-10T18:53:40.384Z{"msg":"[1]R flush_numbers[0..12]: [125, 133, 132, 133, 133, 133, 130, 133, 104, 133, 133, 131]","v":0,"name":"crucible"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
96182023-12-10T18:53:40.385Z,"level":30,"time":"2023-12-10T18:53:39.943324104Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96192023-12-10T18:53:40.385Z{"msg":"[1]R generation[0..12]: [72, 77, 76, 77, 77, 77, 74, 77, 62, 77, 77, 75]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.943389182Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96202023-12-10T18:53:40.385Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.943416426Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96212023-12-10T18:53:40.386Z{"msg":"[2]R flush_numbers[0..12]: [125, 133, 132, 133, 133, 133, 130, 133, 104, 133, 133, 131]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.943458707Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96222023-12-10T18:53:40.386Z{"msg":"[2]R generation[0..12]: [72, 77, 76, 77, 77, 77, 74, 77, 62, 77, 77, 75]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.94348211Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96232023-12-10T18:53:40.386Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.94350395Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96242023-12-10T18:53:40.387Z{"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.94352367Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96252023-12-10T18:53:40.387Z{"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.94354355Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96262023-12-10T18:53:40.387Z{"msg":"Next flush: 134","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.943590619Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96272023-12-10T18:53:40.388Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.943615554Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96282023-12-10T18:53:40.388Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.943656898Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96292023-12-10T18:53:40.389Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.943679495Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96302023-12-10T18:53:40.389Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.943699527Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96312023-12-10T18:53:40.389Z{"msg":"d8b84b20-8ce9-4e5b-913e-46c8eaa92421 is now active with session: 29f78388-2fa0-40e3-a5fa-209c0d081f1a","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.943741479Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96322023-12-10T18:53:40.389Z{"msg":"d8b84b20-8ce9-4e5b-913e-46c8eaa92421 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.943764359Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96332023-12-10T18:53:40.389Z{"msg":"Loading write count information from file "/tmp/test_repair_verify.data"
96342023-12-10T18:53:40.389ZNotify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.94381104Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"Run Repair workload
96352023-12-10T18:53:40.389Z}
96362023-12-10T18:53:40.390Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.943835393Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96372023-12-10T18:53:40.391Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.943875772Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96382023-12-10T18:53:40.392Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.943898078Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96392023-12-10T18:53:40.392Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.943937377Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96402023-12-10T18:53:40.392Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":3001/30 Flush
96412023-12-10T18:53:40.392Z,"time":"2023-12-10T18:53:39.943959051Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96422023-12-10T18:53:40.393Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.944002389Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96432023-12-10T18:53:40.393Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.944024062Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96442023-12-10T18:53:40.393Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.944073098Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96452023-12-10T18:53:40.393Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.944094929Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96462023-12-10T18:53:40.393Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:39.944134369Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1474,"session_id":"29f78388-2fa0-40e3-a5fa-209c0d081f1a"}
96472023-12-10T18:53:40.393Z02/30 Write block 195 len 28672 data: 12 11 12 10 9 8 8
96482023-12-10T18:53:40.394Z03/30 Write block 340 len 28672 data: 4 5 7 8 8 7 6
96492023-12-10T18:53:40.394Z04/30 Write block 223 len 40960 data: 9 9 7 9 8 7 7 8 8 8
96502023-12-10T18:53:40.405Z05/30 Read block 239 len 16384
96512023-12-10T18:53:40.405Z06/30 Write block 241 len 32768 data: 12 12 11 10 9 10 8 9
96522023-12-10T18:53:40.406Z07/30 Read block 197 len 24576
96532023-12-10T18:53:40.406Z08/30 Write block 32 len 36864 data: 9 6 6 7 8 7 6 7 6
96542023-12-10T18:53:40.406Z09/30 Read block 585 len 20480
96552023-12-10T18:53:40.407Z10/30 Write block 154 len 28672 data: 7 3 4 7 6 6 6
96562023-12-10T18:53:40.407Z11/30 Read block 25 len 32768
96572023-12-10T18:53:40.407Z12/30 Read block 470 len 28672
96582023-12-10T18:53:40.408Z13/30 Write block 314 len 8192 data: 5 4
96592023-12-10T18:53:40.408Z14/30 Read block 457 len 20480
96602023-12-10T18:53:40.408Z15/30 Write block 128 len 16384 data: 10 9 6 5
96612023-12-10T18:53:40.409Z16/30 Read block 547 len 20480
96622023-12-10T18:53:40.409Z17/30 Read block 424 len 12288
96632023-12-10T18:53:40.409Z18/30 Read block 204 len 24576
96642023-12-10T18:53:40.409Z19/30 Read block 481 len 20480
96652023-12-10T18:53:40.409Z20/30 Flush
96662023-12-10T18:53:40.410Z21/30 Write block 12 len 12288 data: 7 10 11
96672023-12-10T18:53:40.410Z22/30 Flush
96682023-12-10T18:53:40.410Z23/30 Write block 263 len 28672 data: 5 6 7 6 5 6 7
96692023-12-10T18:53:40.411Z24/30 Read block 173 len 36864
96702023-12-10T18:53:40.411Z25/30 Write block 555 len 32768 data: 6 5 2 4 5 5 7 7
96712023-12-10T18:53:40.411Z26/30 Write block 335 len 36864 data: 5 5 6 5 5 5 6 8 9
96722023-12-10T18:53:40.411Z27/30 Read block 409 len 32768
96732023-12-10T18:53:40.412Z28/30 Read block 99 len 8192
96742023-12-10T18:53:40.412Z29/30 Read block 129 len 24576
96752023-12-10T18:53:40.412Z30/30 Write block 45 len 4096 data: 6
96762023-12-10T18:53:40.413Z----------------------------------------------------------------
96772023-12-10T18:53:40.413Z Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30
96782023-12-10T18:53:40.413ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
96792023-12-10T18:53:40.413Z 1 Acked 1000 Flush 0 Sent Done Done false
96802023-12-10T18:53:40.414Z 2 Acked 1001 Write 7 Sent Done Done false
96812023-12-10T18:53:40.414Z 3 Acked 1002 Write 7 Sent Done Done false
96822023-12-10T18:53:40.414Z 4 Acked 1003 Write 10 Sent Done Done false
96832023-12-10T18:53:40.414Z 5 Acked 1004 Read 4 Sent Done Done false
96842023-12-10T18:53:40.415Z 6 Acked 1005 Write 8 Sent Done Done false
96852023-12-10T18:53:40.415Z 7 Acked 1006 Read 6 Sent Done Done false
96862023-12-10T18:53:40.415Z 8 Acked 1007 Write 9 Sent Done Done false
96872023-12-10T18:53:40.416Z 9 Acked 1008 Read 5 Sent Done Done false
96882023-12-10T18:53:40.416Z 10 Acked 1009 Write 7 Sent Done Done false
96892023-12-10T18:53:40.416Z 11 Acked 1010 Read 8 Sent Done Done false
96902023-12-10T18:53:40.416Z 12 Acked 1011 Read 7 Sent Done Done false
96912023-12-10T18:53:40.417Z 13 Acked 1012 Write 2 Sent Done Done false
96922023-12-10T18:53:40.417Z 14 Acked 1013 Read 5 Sent Done Done false
96932023-12-10T18:53:40.417Z 15 Acked 1014 Write 4 Sent Done Done false
96942023-12-10T18:53:40.417Z 16 Acked 1015 Read 5 Sent Done Done false
96952023-12-10T18:53:40.418Z 17 Acked 1016 Read 3 Sent Done Done false
96962023-12-10T18:53:40.418Z 18 Acked 1017 Read 6 Sent Done Done false
96972023-12-10T18:53:40.418Z 19 Acked 1018 Read 5 Sent Done Done false
96982023-12-10T18:53:40.418Z 20 Acked 1019 Flush 0 Sent Done Done false
96992023-12-10T18:53:40.418Z 21 Acked 1020 Write 3 Sent Done Done false
97002023-12-10T18:53:40.419Z 22 Acked 1021 Flush 0 Sent Done Done false
97012023-12-10T18:53:40.419Z 23 Acked 1022 Write 7 Sent Done Done false
97022023-12-10T18:53:40.419Z 24 Acked 1023 Read 9 Sent Done Done false
97032023-12-10T18:53:40.419Z 25 Acked 1024 Write 8 Sent Done Done false
97042023-12-10T18:53:40.419Z 26 Acked 1025 Write 9 Sent Done Done false
97052023-12-10T18:53:40.420Z 27 Acked 1026 Read 8 Sent Done Done false
97062023-12-10T18:53:40.420Z 28 Acked 1027 Read 2 Sent Done Done false
97072023-12-10T18:53:40.420Z 29 Acked 1028 Read 6 Sent Done Done false
97082023-12-10T18:53:40.420Z 30 Acked 1029 Write 1 Sent Sent Sent false
97092023-12-10T18:53:40.420Z STATES DS:0 DS:1 DS:2 TOTAL
97102023-12-10T18:53:40.421Z New 0 0 0 0
97112023-12-10T18:53:40.421Z Sent 30 1 1 32
97122023-12-10T18:53:40.421Z Done 0 29 29 58
97132023-12-10T18:53:40.421Z Skipped 0 0 0 0
97142023-12-10T18:53:40.422Z Error 0 0 0 0
97152023-12-10T18:53:40.422ZLast Flush: 0 1021 1021
97162023-12-10T18:53:40.422ZDownstairs last five completed:
97172023-12-10T18:53:40.422ZUpstairs last five completed: 30 29 28 27 26 25
97182023-12-10T18:53:40.423ZWrote out file "/tmp/test_repair_verify.data"
97192023-12-10T18:53:40.423Z
97202023-12-10T18:53:40.423ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
97212023-12-10T18:53:40.424ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
97222023-12-10T18:53:40.424Z 0 000-019 72 78 78 125 136 136 F F F <---
97232023-12-10T18:53:40.424Z 1 020-039 77 78 78 133 135 135 F F F <---
97242023-12-10T18:53:40.424Z 2 040-059 76 78 78 132 135 135 F T T <---
97252023-12-10T18:53:40.424Z 3 060-079 77 77 77 133 133 133 F F F
97262023-12-10T18:53:40.425Z 4 080-099 77 77 77 133 133 133 F F F
97272023-12-10T18:53:40.425Z 5 100-119 77 77 77 133 133 133 F F F
97282023-12-10T18:53:40.425Z 6 120-139 74 78 78 130 135 135 F F F <---
97292023-12-10T18:53:40.425Z 7 140-159 77 78 78 133 135 135 F F F <---
97302023-12-10T18:53:40.426Z 8 160-179 62 78 78 104 135 135 F F F <---
97312023-12-10T18:53:40.426Z 9 180-199 77 78 78 133 135 135 F F F <---
97322023-12-10T18:53:40.426Z 10 200-219 77 78 78 133 135 135 F F F <---
97332023-12-10T18:53:40.426Z 11 220-239 75 78 78 131 135 135 F F F <---
97342023-12-10T18:53:40.427Z 12 240-259 75 78 78 131 135 135 F F F <---
97352023-12-10T18:53:40.427Z 13 260-279 77 77 77 133 133 133 F T T <---
97362023-12-10T18:53:40.427Z 14 280-299 74 74 74 130 130 130 F F F
97372023-12-10T18:53:40.427Z 15 300-319 70 78 78 122 135 135 F F F <---
97382023-12-10T18:53:40.427Z 16 320-339 77 77 77 133 133 133 F T T <---
97392023-12-10T18:53:40.428Z 17 340-359 76 78 78 132 135 135 F T T <---
97402023-12-10T18:53:40.428Z 18 360-379 76 76 76 132 132 132 F F F
97412023-12-10T18:53:40.428Z 19 380-399 77 77 77 133 133 133 F F F
97422023-12-10T18:53:40.429Z 20 400-419 72 72 72 127 127 127 F F F
97432023-12-10T18:53:40.429Z 21 420-439 59 59 59 98 98 98 F F F
97442023-12-10T18:53:40.429Z 22 440-459 74 74 74 130 130 130 F F F
97452023-12-10T18:53:40.429Z 23 460-479 72 72 72 125 125 125 F F F
97462023-12-10T18:53:40.430Z 24 480-499 75 75 75 131 131 131 F F F
97472023-12-10T18:53:40.431Z 25 500-519 75 75 75 131 131 131 F F F
97482023-12-10T18:53:40.432Z 26 520-539 77 77 77 133 133 133 F F F
97492023-12-10T18:53:40.432Z 27 540-559 75 75 75 131 131 131 F T T <---
97502023-12-10T18:53:40.440Z 28 560-579 75 75 75 131 131 131 F T T <---
97512023-12-10T18:53:40.441Z 29 580-599 74 74 74 130 130 130 F F F
97522023-12-10T18:53:40.441ZMax gen: 78, Max flush: 136
97532023-12-10T18:53:40.441ZError: Difference in extent metadata found!
97542023-12-10T18:53:40.442ZA Difference in extent metadata is expected here
97552023-12-10T18:53:40.442ZOn loop 39
97562023-12-10T18:53:40.442Z
97572023-12-10T18:53:40.443ZVerifying data now
97582023-12-10T18:53:45.538ZNo change after verify
97592023-12-10T18:53:45.538ZLoop: 39 Downstairs dump after verify (and repair):
97602023-12-10T18:53:45.558ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
97612023-12-10T18:53:45.558Z 0 000-019 78 78 78 136 136 136 F F F
97622023-12-10T18:53:45.575Z 1 020-039 78 78 78 135 135 135 F F F
97632023-12-10T18:53:45.575Z 2 040-059 79 79 79 137 137 137 F F F
97642023-12-10T18:53:45.577Z 3 060-079 77 77 77 133 133 133 F F F
97652023-12-10T18:53:45.577Z 4 080-099 77 77 77 133 133 133 F F F
97662023-12-10T18:53:45.577Z 5 100-119 77 77 77 133 133 133 F F F
97672023-12-10T18:53:45.578Z 6 120-139 78 78 78 135 135 135 F F F
97682023-12-10T18:53:45.578Z 7 140-159 78 78 78 135 135 135 F F F
97692023-12-10T18:53:45.578Z 8 160-179 78 78 78 135 135 135 F F F
97702023-12-10T18:53:45.579Z 9 180-199 78 78 78 135 135 135 F F F
97712023-12-10T18:53:45.579Z 10 200-219 78 78 78 135 135 135 F F F
97722023-12-10T18:53:45.580Z 11 220-239 78 78 78 135 135 135 F F F
97732023-12-10T18:53:45.580Z 12 240-259 78 78 78 135 135 135 F F F
97742023-12-10T18:53:45.580Z 13 260-279 79 79 79 137 137 137 F F F
97752023-12-10T18:53:45.581Z 14 280-299 74 74 74 130 130 130 F F F
97762023-12-10T18:53:45.581Z 15 300-319 78 78 78 135 135 135 F F F
97772023-12-10T18:53:45.581Z 16 320-339 79 79 79 137 137 137 F F F
97782023-12-10T18:53:45.581Z 17 340-359 79 79 79 137 137 137 F F F
97792023-12-10T18:53:45.581Z 18 360-379 76 76 76 132 132 132 F F F
97802023-12-10T18:53:45.582Z 19 380-399 77 77 77 133 133 133 F F F
97812023-12-10T18:53:45.582Z 20 400-419 72 72 72 127 127 127 F F F
97822023-12-10T18:53:45.582Z 21 420-439 59 59 59 98 98 98 F F F
97832023-12-10T18:53:45.582Z 22 440-459 74 74 74 130 130 130 F F F
97842023-12-10T18:53:45.582Z 23 460-479 72 72 72 125 125 125 F F F
97852023-12-10T18:53:45.583Z 24 480-499 75 75 75 131 131 131 F F F
97862023-12-10T18:53:45.583Z 25 500-519 75 75 75 131 131 131 F F F
97872023-12-10T18:53:45.583Z 26 520-539 77 77 77 133 133 133 F F F
97882023-12-10T18:53:45.583Z 27 540-559 79 79 79 137 137 137 F F F
97892023-12-10T18:53:45.583Z 28 560-579 79 79 79 137 137 137 F F F
97902023-12-10T18:53:45.584Z 29 580-599 74 74 74 130 130 130 F F F
97912023-12-10T18:53:45.584ZMax gen: 79, Max flush: 137
97922023-12-10T18:53:45.584Z
97932023-12-10T18:53:45.584ZBegin loop 40
97942023-12-10T18:53:45.584ZDownstairs to restart: 0
97952023-12-10T18:53:45.591Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.58676869Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482}
97962023-12-10T18:53:45.591Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.58717767Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482}
97972023-12-10T18:53:45.592Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.58720527Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482}
97982023-12-10T18:53:45.592Z{"msg":"Crucible 6cb601c1-3748-418f-a268-42d4cbecf7be has session id: 21975ae3-de9a-4629-82eb-694cfc3fc866","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.587249039Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
97992023-12-10T18:53:45.592Z{"msg":"Upstairs opts: Upstairs UUID: 6cb601c1-3748-418f-a268-42d4cbecf7be, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.587286517Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98002023-12-10T18:53:45.593Z{"msg":"Crucible stats registered with UUID: 6cb601c1-3748-418f-a268-42d4cbecf7be","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.587355356Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98012023-12-10T18:53:45.593ZCrucible runtime is spawned
98022023-12-10T18:53:45.593ZThe guest has requested activation with gen:80
98032023-12-10T18:53:45.594Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.587597056Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866","task":"up_listen"}
98042023-12-10T18:53:45.594Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.587661314Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"looper":"2","session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98052023-12-10T18:53:45.594Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.587695149Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98062023-12-10T18:53:45.594Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.587754991Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98072023-12-10T18:53:45.594Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.587990496Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"looper":"0","session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98082023-12-10T18:53:45.594Z{"msg":"[2] 6cb601c1-3748-418f-a268-42d4cbecf7be looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.588054693Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"looper":"2","session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98092023-12-10T18:53:45.594Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.588164974Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"looper":"1","session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98102023-12-10T18:53:45.594Z{"msg":"[0] 6cb601c1-3748-418f-a268-42d4cbecf7be looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.588218212Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"looper":"0","session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98112023-12-10T18:53:45.594Z{"msg":"[1] 6cb601c1-3748-418f-a268-42d4cbecf7be looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.588415126Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"looper":"1","session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98122023-12-10T18:53:45.594Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.588853909Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98132023-12-10T18:53:45.594Z{"msg":"6cb601c1-3748-418f-a268-42d4cbecf7be active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.588911371Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98142023-12-10T18:53:45.594Z{"msg":"Set desired generation to :80","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.58893659Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98152023-12-10T18:53:45.594Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.589313956Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98162023-12-10T18:53:45.595Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.589397481Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98172023-12-10T18:53:45.595Z{"msg":"[2] 6cb601c1-3748-418f-a268-42d4cbecf7be (21975ae3-de9a-4629-82eb-694cfc3fc866) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.58970688Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98182023-12-10T18:53:45.595Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.589793361Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98192023-12-10T18:53:45.595Z{"msg":"[2] client is_active_req TRUE, promote! session 21975ae3-de9a-4629-82eb-694cfc3fc866","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.589866014Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98202023-12-10T18:53:45.595Z{"msg":"[1] 6cb601c1-3748-418f-a268-42d4cbecf7be (21975ae3-de9a-4629-82eb-694cfc3fc866) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.589938705Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98212023-12-10T18:53:45.595Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.5900083Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98222023-12-10T18:53:45.595Z{"msg":"[1] client is_active_req TRUE, promote! session 21975ae3-de9a-4629-82eb-694cfc3fc866","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.59004836Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98232023-12-10T18:53:45.595Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.590391356Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98242023-12-10T18:53:45.595Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.590462403Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98252023-12-10T18:53:45.595Z{"msg":"6cb601c1-3748-418f-a268-42d4cbecf7be New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.590489114Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98262023-12-10T18:53:45.595Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.590677936Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98272023-12-10T18:53:45.595Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.590726279Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98282023-12-10T18:53:45.595Z{"msg":"[0] 6cb601c1-3748-418f-a268-42d4cbecf7be (21975ae3-de9a-4629-82eb-694cfc3fc866) New WaitActive WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.590750739Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98292023-12-10T18:53:45.595Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.590784495Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98302023-12-10T18:53:45.595Z{"msg":"[0] client is_active_req TRUE, promote! session 21975ae3-de9a-4629-82eb-694cfc3fc866","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.590826126Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98312023-12-10T18:53:45.595Z{"msg":"6cb601c1-3748-418f-a268-42d4cbecf7be WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.590900694Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98322023-12-10T18:53:45.596Z{"msg":"[2] 6cb601c1-3748-418f-a268-42d4cbecf7be (21975ae3-de9a-4629-82eb-694cfc3fc866) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.591231277Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98332023-12-10T18:53:45.596Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.59134406Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98342023-12-10T18:53:45.596Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:45.591462029Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98352023-12-10T18:53:45.596Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.591515255Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98362023-12-10T18:53:45.596Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.591553603Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98372023-12-10T18:53:45.596Z{"msg":"6cb601c1-3748-418f-a268-42d4cbecf7be WaitActive WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.591586784Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98382023-12-10T18:53:45.596Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.591618871Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98392023-12-10T18:53:45.596Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.59165187Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98402023-12-10T18:53:45.597Z{"msg":"[1] 6cb601c1-3748-418f-a268-42d4cbecf7be (21975ae3-de9a-4629-82eb-694cfc3fc866) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.591712852Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98412023-12-10T18:53:45.597Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.59175243Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98422023-12-10T18:53:45.597Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:45.591786181Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98432023-12-10T18:53:45.598Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.591841031Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98442023-12-10T18:53:45.598Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.591880176Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98452023-12-10T18:53:45.598Z{"msg":"6cb601c1-3748-418f-a268-42d4cbecf7be WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.591937097Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98462023-12-10T18:53:45.598Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.591976362Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98472023-12-10T18:53:45.598Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.592034089Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98482023-12-10T18:53:45.598Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.592074194Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98492023-12-10T18:53:45.598Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.592135539Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98502023-12-10T18:53:45.598Z{"msg":"6cb601c1-3748-418f-a268-42d4cbecf7be WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.592175754Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98512023-12-10T18:53:45.603Z{"msg":"[0] 6cb601c1-3748-418f-a268-42d4cbecf7be (21975ae3-de9a-4629-82eb-694cfc3fc866) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.592697486Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98522023-12-10T18:53:45.603Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.592768333Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98532023-12-10T18:53:45.603Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:45.592800404Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98542023-12-10T18:53:45.603Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.592856385Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98552023-12-10T18:53:45.603Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.592884568Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98562023-12-10T18:53:45.603Z{"msg":"6cb601c1-3748-418f-a268-42d4cbecf7be WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.592940254Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98572023-12-10T18:53:45.603Z{"msg":"[0]R flush_numbers[0..12]: [136, 135, 137, 133, 133, 133, 135, 135, 135, 135, 135, 135]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.592973076Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98582023-12-10T18:53:45.604Z{"msg":"[0]R generation[0..12]: [78, 78, 79, 77, 77, 77, 78, 78, 78, 78, 78, 78]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.593029933Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98592023-12-10T18:53:45.604Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.593061939Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98602023-12-10T18:53:45.604Z{"msg":"[1]R flush_numbers[0..12]: [136, 135, 137, 133, 133, 133, 135, 135, 135, 135, 135, 135]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.59309721Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98612023-12-10T18:53:45.604Z{"msg":"[1]R generation[0..12]: [78, 78, 79, 77, 77, 77, 78, 78, 78, 78, 78, 78]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.593124418Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98622023-12-10T18:53:45.604Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.593233552Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98632023-12-10T18:53:45.606Z{"msg":"[2]R flush_numbers[0..12]: [136, 135, 137, 133, 133, 133, 135, 135, 135, 135, 135, 135]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:80
98642023-12-10T18:53:45.606ZWait for a query_work_queue command to finish before sending IO
98652023-12-10T18:53:45.606Z,"time":"2023-12-10T18:53:45.59332779Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98662023-12-10T18:53:45.606Z{"msg":"[2]R generation[0..12]: [78, 78, 79, 77, 77, 77, 78, 78, 78, 78, 78, 78]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.593478488Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98672023-12-10T18:53:45.607Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.593513727Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98682023-12-10T18:53:45.607Z{"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.593590701Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98692023-12-10T18:53:45.607Z{"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.593667142Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98702023-12-10T18:53:45.607Z{"msg":"Next flush: 138","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.593691653Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98712023-12-10T18:53:45.607Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.593747102Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98722023-12-10T18:53:45.607Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.593773134Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98732023-12-10T18:53:45.607Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.59383382Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98742023-12-10T18:53:45.607Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.59392345Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98752023-12-10T18:53:45.607Z{"msg":"6cb601c1-3748-418f-a268-42d4cbecf7be is now active with session: 21975ae3-de9a-4629-82eb-694cfc3fc866","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.593961648Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98762023-12-10T18:53:45.607Z{"msg":"6cb601c1-3748-418f-a268-42d4cbecf7be Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.593983677Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98772023-12-10T18:53:45.608Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.594010167Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98782023-12-10T18:53:45.608Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.594064103Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98792023-12-10T18:53:45.608Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.594087891Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98802023-12-10T18:53:45.608Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.594145754Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98812023-12-10T18:53:45.608Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.594174571Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98822023-12-10T18:53:45.608Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.594239281Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98832023-12-10T18:53:45.608Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
98842023-12-10T18:53:45.608Z,"time":"2023-12-10T18:53:45.594316275Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98852023-12-10T18:53:45.608Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.594433537Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98862023-12-10T18:53:45.609Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.59446623Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98872023-12-10T18:53:45.609Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.594519873Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98882023-12-10T18:53:45.610Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:45.594546233Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1482,"session_id":"21975ae3-de9a-4629-82eb-694cfc3fc866"}
98892023-12-10T18:53:45.610ZLoading write count information from file "/tmp/test_repair_verify.data"
98902023-12-10T18:53:45.610ZRun Repair workload
98912023-12-10T18:53:45.610Z01/30 Write block 57 len 16384 data: 5 4 4 3
98922023-12-10T18:53:45.610Z02/30 Read block 222 len 32768
98932023-12-10T18:53:45.610Z03/30 Read block 402 len 20480
98942023-12-10T18:53:45.610Z04/30 Write block 521 len 24576 data: 5 4 4 6 6 6
98952023-12-10T18:53:45.610Z05/30 Read block 182 len 28672
98962023-12-10T18:53:45.610Z06/30 Read block 517 len 40960
98972023-12-10T18:53:45.630Z07/30 Write block 129 len 32768 data: 10 7 6 2 3 3 3 3
98982023-12-10T18:53:45.630Z08/30 Flush
98992023-12-10T18:53:45.631Z09/30 Write block 445 len 32768 data: 6 5 5 6 6 6 7 7
99002023-12-10T18:53:45.631Z10/30 Read block 413 len 8192
99012023-12-10T18:53:45.631Z11/30 Read block 247 len 4096
99022023-12-10T18:53:45.632Z12/30 Flush
99032023-12-10T18:53:45.645Z13/30 Write block 121 len 12288 data: 11 11 13
99042023-12-10T18:53:45.645Z14/30 Write block 260 len 28672 data: 2 3 4 6 7 8 7
99052023-12-10T18:53:45.645Z15/30 Read block 593 len 12288
99062023-12-10T18:53:45.645Z16/30 Write block 289 len 12288 data: 7 4 6
99072023-12-10T18:53:45.646Z17/30 Write block 166 len 16384 data: 11 11 8 7
99082023-12-10T18:53:45.647Z18/30 Write block 566 len 20480 data: 9 8 7 8 7
99092023-12-10T18:53:45.647Z19/30 Read block 220 len 28672
99102023-12-10T18:53:45.648Z20/30 Read block 75 len 4096
99112023-12-10T18:53:45.648Z21/30 Write block 339 len 36864 data: 6 6 7 9 10 9 8 7 8
99122023-12-10T18:53:45.648Z22/30 Read block 466 len 40960
99132023-12-10T18:53:45.648Z23/30 Read block 248 len 24576
99142023-12-10T18:53:45.649Z24/30 Write block 557 len 32768 data: 3 5 6 6 8 8 7 8
99152023-12-10T18:53:45.649Z25/30 Flush
99162023-12-10T18:53:45.659Z26/30 Write block 340 len 20480 data: 7 8 10 11 10
99172023-12-10T18:53:45.660Z27/30 Read block 192 len 40960
99182023-12-10T18:53:45.660Z28/30 Read block 352 len 40960
99192023-12-10T18:53:45.660Z29/30 Write block 349 len 32768 data: 8 7 7 8 10 8 6 7
99202023-12-10T18:53:45.660Z30/30 Read block 107 len 40960
99212023-12-10T18:53:45.660Z----------------------------------------------------------------
99222023-12-10T18:53:45.661Z Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:30
99232023-12-10T18:53:45.661ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
99242023-12-10T18:53:45.661Z 1 Acked 1000 Write 4 Done Done Done false
99252023-12-10T18:53:45.661Z 2 Acked 1001 Read 8 Done Done Done false
99262023-12-10T18:53:45.661Z 3 Acked 1002 Read 5 Done Done Done false
99272023-12-10T18:53:45.662Z 4 Acked 1003 Write 6 Done Done Done false
99282023-12-10T18:53:45.662Z 5 Acked 1004 Read 7 Sent Done Done false
99292023-12-10T18:53:45.662Z 6 Acked 1005 Read 10 Sent Done Done false
99302023-12-10T18:53:45.662Z 7 Acked 1006 Write 8 Sent Done Done false
99312023-12-10T18:53:45.662Z 8 Acked 1007 Flush 0 Sent Done Done false
99322023-12-10T18:53:45.662Z 9 Acked 1008 Write 8 Sent Done Done false
99332023-12-10T18:53:45.663Z 10 Acked 1009 Read 2 Sent Done Done false
99342023-12-10T18:53:45.663Z 11 Acked 1010 Read 1 Sent Done Done false
99352023-12-10T18:53:45.663Z 12 Acked 1011 Flush 0 Sent Done Done false
99362023-12-10T18:53:45.663Z 13 Acked 1012 Write 3 Sent Done Done false
99372023-12-10T18:53:45.663Z 14 Acked 1013 Write 7 Sent Done Done false
99382023-12-10T18:53:45.664Z 15 Acked 1014 Read 3 Sent Done Done false
99392023-12-10T18:53:45.664Z 16 Acked 1015 Write 3 Sent Done Done false
99402023-12-10T18:53:45.664Z 17 Acked 1016 Write 4 Sent Done Done false
99412023-12-10T18:53:45.664Z 18 Acked 1017 Write 5 Sent Done Done false
99422023-12-10T18:53:45.665Z 19 Acked 1018 Read 7 Sent Done Done false
99432023-12-10T18:53:45.665Z 20 Acked 1019 Read 1 Sent Done Done false
99442023-12-10T18:53:45.665Z 21 Acked 1020 Write 9 Sent Done Done false
99452023-12-10T18:53:45.665Z 22 Acked 1021 Read 10 Sent Done Done false
99462023-12-10T18:53:45.666Z 23 Acked 1022 Read 6 Sent Done Done false
99472023-12-10T18:53:45.666Z 24 Acked 1023 Write 8 Sent Done Done false
99482023-12-10T18:53:45.666Z 25 Acked 1024 Flush 0 Sent Done Done false
99492023-12-10T18:53:45.668Z 26 Acked 1025 Write 5 Sent Done Done false
99502023-12-10T18:53:45.668Z 27 Acked 1026 Read 10 Sent Done Done false
99512023-12-10T18:53:45.669Z 28 Acked 1027 Read 10 Sent Done Done false
99522023-12-10T18:53:45.669Z 29 Acked 1028 Write 8 Sent Done Done false
99532023-12-10T18:53:45.669Z 30 Acked 1029 Read 10 Sent Done Done false
99542023-12-10T18:53:45.669Z STATES DS:0 DS:1 DS:2 TOTAL
99552023-12-10T18:53:45.669Z New 0 0 0 0
99562023-12-10T18:53:45.670Z Sent 26 0 0 26
99572023-12-10T18:53:45.670Z Done 4 30 30 64
99582023-12-10T18:53:45.670Z Skipped 0 0 0 0
99592023-12-10T18:53:45.670Z Error 0 0 0 0
99602023-12-10T18:53:45.671ZLast Flush: 0 1024 1024
99612023-12-10T18:53:45.671ZDownstairs last five completed:
99622023-12-10T18:53:45.671ZUpstairs last five completed: 30 29 28 27 26 25
99632023-12-10T18:53:45.671ZWrote out file "/tmp/test_repair_verify.data"
99642023-12-10T18:53:45.676Z
99652023-12-10T18:53:45.676ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
99662023-12-10T18:53:45.688ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
99672023-12-10T18:53:45.688Z 0 000-019 78 78 78 136 136 136 F F F
99682023-12-10T18:53:45.692Z 1 020-039 78 78 78 135 135 135 F F F
99692023-12-10T18:53:45.692Z 2 040-059 79 80 80 137 138 138 T F F <---
99702023-12-10T18:53:45.692Z 3 060-079 77 80 80 133 138 138 T F F <---
99712023-12-10T18:53:45.692Z 4 080-099 77 77 77 133 133 133 F F F
99722023-12-10T18:53:45.692Z 5 100-119 77 77 77 133 133 133 F F F
99732023-12-10T18:53:45.692Z 6 120-139 78 80 80 135 140 140 F F F <---
99742023-12-10T18:53:45.692Z 7 140-159 78 78 78 135 135 135 F F F
99752023-12-10T18:53:45.692Z 8 160-179 78 80 80 135 140 140 F F F <---
99762023-12-10T18:53:45.692Z 9 180-199 78 78 78 135 135 135 F F F
99772023-12-10T18:53:45.692Z 10 200-219 78 78 78 135 135 135 F F F
99782023-12-10T18:53:45.692Z 11 220-239 78 78 78 135 135 135 F F F
99792023-12-10T18:53:45.692Z 12 240-259 78 78 78 135 135 135 F F F
99802023-12-10T18:53:45.692Z 13 260-279 79 80 80 137 140 140 F F F <---
99812023-12-10T18:53:45.692Z 14 280-299 74 80 80 130 140 140 F F F <---
99822023-12-10T18:53:45.692Z 15 300-319 78 78 78 135 135 135 F F F
99832023-12-10T18:53:45.692Z 16 320-339 79 80 80 137 140 140 F F F <---
99842023-12-10T18:53:45.693Z 17 340-359 79 80 80 137 140 140 F T T <---
99852023-12-10T18:53:45.693Z 18 360-379 76 76 76 132 132 132 F F F
99862023-12-10T18:53:45.693Z 19 380-399 77 77 77 133 133 133 F F F
99872023-12-10T18:53:45.693Z 20 400-419 72 72 72 127 127 127 F F F
99882023-12-10T18:53:45.693Z 21 420-439 59 59 59 98 98 98 F F F
99892023-12-10T18:53:45.693Z 22 440-459 74 80 80 130 139 139 F F F <---
99902023-12-10T18:53:45.693Z 23 460-479 72 72 72 125 125 125 F F F
99912023-12-10T18:53:45.693Z 24 480-499 75 75 75 131 131 131 F F F
99922023-12-10T18:53:45.693Z 25 500-519 75 75 75 131 131 131 F F F
99932023-12-10T18:53:45.693Z 26 520-539 77 80 80 133 138 138 T F F <---
99942023-12-10T18:53:45.693Z 27 540-559 79 80 80 137 140 140 F F F <---
99952023-12-10T18:53:45.693Z 28 560-579 79 80 80 137 140 140 F F F <---
99962023-12-10T18:53:45.693Z 29 580-599 74 74 74 130 130 130 F F F
99972023-12-10T18:53:45.693ZMax gen: 80, Max flush: 140
99982023-12-10T18:53:45.693ZError: Difference in extent metadata found!
99992023-12-10T18:53:45.696ZA Difference in extent metadata is expected here
100002023-12-10T18:53:45.696ZOn loop 40
100012023-12-10T18:53:45.696Z
100022023-12-10T18:53:45.699ZVerifying data now
100032023-12-10T18:53:51.874ZNo change after verify
100042023-12-10T18:53:51.875ZLoop: 40 Downstairs dump after verify (and repair):
100052023-12-10T18:53:51.897ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
100062023-12-10T18:53:51.897Z 0 000-019 78 78 78 136 136 136 F F F
100072023-12-10T18:53:51.900Z 1 020-039 78 78 78 135 135 135 F F F
100082023-12-10T18:53:51.900Z 2 040-059 80 80 80 138 138 138 F F F
100092023-12-10T18:53:51.900Z 3 060-079 80 80 80 138 138 138 F F F
100102023-12-10T18:53:51.900Z 4 080-099 77 77 77 133 133 133 F F F
100112023-12-10T18:53:51.900Z 5 100-119 77 77 77 133 133 133 F F F
100122023-12-10T18:53:51.900Z 6 120-139 80 80 80 140 140 140 F F F
100132023-12-10T18:53:51.900Z 7 140-159 78 78 78 135 135 135 F F F
100142023-12-10T18:53:51.901Z 8 160-179 80 80 80 140 140 140 F F F
100152023-12-10T18:53:51.901Z 9 180-199 78 78 78 135 135 135 F F F
100162023-12-10T18:53:51.901Z 10 200-219 78 78 78 135 135 135 F F F
100172023-12-10T18:53:51.901Z 11 220-239 78 78 78 135 135 135 F F F
100182023-12-10T18:53:51.901Z 12 240-259 78 78 78 135 135 135 F F F
100192023-12-10T18:53:51.901Z 13 260-279 80 80 80 140 140 140 F F F
100202023-12-10T18:53:51.901Z 14 280-299 80 80 80 140 140 140 F F F
100212023-12-10T18:53:51.901Z 15 300-319 78 78 78 135 135 135 F F F
100222023-12-10T18:53:51.901Z 16 320-339 80 80 80 140 140 140 F F F
100232023-12-10T18:53:51.901Z 17 340-359 81 81 81 141 141 141 F F F
100242023-12-10T18:53:51.901Z 18 360-379 76 76 76 132 132 132 F F F
100252023-12-10T18:53:51.901Z 19 380-399 77 77 77 133 133 133 F F F
100262023-12-10T18:53:51.901Z 20 400-419 72 72 72 127 127 127 F F F
100272023-12-10T18:53:51.901Z 21 420-439 59 59 59 98 98 98 F F F
100282023-12-10T18:53:51.901Z 22 440-459 80 80 80 139 139 139 F F F
100292023-12-10T18:53:51.901Z 23 460-479 72 72 72 125 125 125 F F F
100302023-12-10T18:53:51.901Z 24 480-499 75 75 75 131 131 131 F F F
100312023-12-10T18:53:51.902Z 25 500-519 75 75 75 131 131 131 F F F
100322023-12-10T18:53:51.902Z 26 520-539 80 80 80 138 138 138 F F F
100332023-12-10T18:53:51.902Z 27 540-559 80 80 80 140 140 140 F F F
100342023-12-10T18:53:51.902Z 28 560-579 80 80 80 140 140 140 F F F
100352023-12-10T18:53:51.902Z 29 580-599 74 74 74 130 130 130 F F F
100362023-12-10T18:53:51.902ZMax gen: 81, Max flush: 141
100372023-12-10T18:53:51.905Z
100382023-12-10T18:53:51.905ZBegin loop 41
100392023-12-10T18:53:51.905ZDownstairs to restart: 2
100402023-12-10T18:53:51.937Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.935825818Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490}
100412023-12-10T18:53:51.937Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.936202075Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490}
100422023-12-10T18:53:51.957Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.936253116Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490}
100432023-12-10T18:53:51.957Z{"msg":"Crucible 0fdedeec-c218-485b-8d12-321532e9dfc0 has session id: f1f1555b-0857-4ea7-a501-e34e30f96649","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.936301515Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100442023-12-10T18:53:51.957Z{"msg":"Upstairs opts: Upstairs UUID: 0fdedeec-c218-485b-8d12-321532e9dfc0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.936346481Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100452023-12-10T18:53:51.958Z{"msg":"Crucible runtime is spawned
100462023-12-10T18:53:51.958ZCrucible stats registered with UUID: 0fdedeec-c218-485b-8d12-321532e9dfc0","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.93641803Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100472023-12-10T18:53:51.958ZThe guest has requested activation with gen:82
100482023-12-10T18:53:51.958Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.936633034Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"looper":"2","session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100492023-12-10T18:53:51.958Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.936697511Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"looper":"1","session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100502023-12-10T18:53:51.959Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.936738908Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"looper":"0","session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100512023-12-10T18:53:51.959Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.936778306Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649","task":"up_listen"}
100522023-12-10T18:53:51.959Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.936980901Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100532023-12-10T18:53:51.959Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.937063402Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100542023-12-10T18:53:51.959Z{"msg":"0fdedeec-c218-485b-8d12-321532e9dfc0 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.937154421Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100552023-12-10T18:53:51.959Z{"msg":"Set desired generation to :82","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.937208428Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100562023-12-10T18:53:51.959Z{"msg":"[0] 0fdedeec-c218-485b-8d12-321532e9dfc0 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.937233365Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"looper":"0","session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100572023-12-10T18:53:51.959Z{"msg":"[1] 0fdedeec-c218-485b-8d12-321532e9dfc0 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.937256568Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"looper":"1","session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100582023-12-10T18:53:51.959Z{"msg":"[2] 0fdedeec-c218-485b-8d12-321532e9dfc0 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.937299441Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"looper":"2","session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100592023-12-10T18:53:51.959Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.937323712Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100602023-12-10T18:53:51.959Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.937649071Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100612023-12-10T18:53:51.959Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.937748307Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100622023-12-10T18:53:51.959Z{"msg":"[0] 0fdedeec-c218-485b-8d12-321532e9dfc0 (f1f1555b-0857-4ea7-a501-e34e30f96649) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.937996879Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100632023-12-10T18:53:51.959Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.938055106Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100642023-12-10T18:53:51.959Z{"msg":"[0] client is_active_req TRUE, promote! session f1f1555b-0857-4ea7-a501-e34e30f96649","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.938130639Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100652023-12-10T18:53:51.959Z{"msg":"[1] 0fdedeec-c218-485b-8d12-321532e9dfc0 (f1f1555b-0857-4ea7-a501-e34e30f96649) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.938193007Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100662023-12-10T18:53:51.960Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.938238593Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100672023-12-10T18:53:51.960Z{"msg":"[1] client is_active_req TRUE, promote! session f1f1555b-0857-4ea7-a501-e34e30f96649","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.938280431Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100682023-12-10T18:53:51.960Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.938970764Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100692023-12-10T18:53:51.960Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.939044765Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100702023-12-10T18:53:51.960Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.939143341Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100712023-12-10T18:53:51.960Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.939193625Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100722023-12-10T18:53:51.960Z{"msg":"0fdedeec-c218-485b-8d12-321532e9dfc0 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.939305311Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100732023-12-10T18:53:51.960Z{"msg":"0fdedeec-c218-485b-8d12-321532e9dfc0 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.939336088Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100742023-12-10T18:53:51.960Z{"msg":"[2] 0fdedeec-c218-485b-8d12-321532e9dfc0 (f1f1555b-0857-4ea7-a501-e34e30f96649) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.93939007Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100752023-12-10T18:53:51.960Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.93941432Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100762023-12-10T18:53:51.960Z{"msg":"[2] client is_active_req TRUE, promote! session f1f1555b-0857-4ea7-a501-e34e30f96649","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.939435996Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100772023-12-10T18:53:51.960Z{"msg":"[0] 0fdedeec-c218-485b-8d12-321532e9dfc0 (f1f1555b-0857-4ea7-a501-e34e30f96649) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.940000337Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100782023-12-10T18:53:51.960Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.940057916Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100792023-12-10T18:53:51.960Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:51.940082136Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100802023-12-10T18:53:51.960Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.940171431Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100812023-12-10T18:53:51.960Z{"msg":"[1] 0fdedeec-c218-485b-8d12-321532e9dfc0 (f1f1555b-0857-4ea7-a501-e34e30f96649) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.94024593Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100822023-12-10T18:53:51.961Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.940312952Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100832023-12-10T18:53:51.961Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:51.940341641Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100842023-12-10T18:53:51.961Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.940413774Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100852023-12-10T18:53:51.961Z{"msg":"0fdedeec-c218-485b-8d12-321532e9dfc0 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.940445578Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100862023-12-10T18:53:51.961Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.940500308Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100872023-12-10T18:53:51.961Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.940527499Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100882023-12-10T18:53:51.961Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.940581279Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100892023-12-10T18:53:51.962Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.940606214Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100902023-12-10T18:53:51.962Z{"msg":"0fdedeec-c218-485b-8d12-321532e9dfc0 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.940637005Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100912023-12-10T18:53:51.962Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.940688508Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100922023-12-10T18:53:51.962Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.9407128Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100932023-12-10T18:53:51.962Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.940769898Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100942023-12-10T18:53:51.962Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.940805604Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100952023-12-10T18:53:51.963Z{"msg":"0fdedeec-c218-485b-8d12-321532e9dfc0 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.940861377Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100962023-12-10T18:53:51.963Z{"msg":"[2] 0fdedeec-c218-485b-8d12-321532e9dfc0 (f1f1555b-0857-4ea7-a501-e34e30f96649) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.941073058Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100972023-12-10T18:53:51.963Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.941203773Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100982023-12-10T18:53:51.963Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:51.941290999Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
100992023-12-10T18:53:51.963Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.94136089Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101002023-12-10T18:53:51.963Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.941436161Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490The guest has finished waiting for activation with:82
101012023-12-10T18:53:51.963ZWait for a query_work_queue command to finish before sending IO
101022023-12-10T18:53:51.963Z,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101032023-12-10T18:53:51.963Z{"msg":"0fdedeec-c218-485b-8d12-321532e9dfc0 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.94162623Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101042023-12-10T18:53:51.963Z{"msg":"[0]R flush_numbers[0..12]: [136, 135, 138, 138, 133, 133, 140, 135, 140, 135, 135, 135]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.941675619Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101052023-12-10T18:53:51.963Z{"msg":"[0]R generation[0..12]: [78, 78, 80, 80, 77, 77, 80, 78, 80, 78, 78, 78]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.941718832Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101062023-12-10T18:53:51.965Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.941766689Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101072023-12-10T18:53:51.965Z{"msg":"[1]R flush_numbers[0..12]: [136, 135, 138, 138, 133, 133, 140, 135, 140, 135, 135, 135]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.941810701Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101082023-12-10T18:53:51.965Z{"msg":"[1]R generation[0..12]: [78, 78, 80, 80, 77, 77, 80, 78, 80, 78, 78, 78]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.941945529Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101092023-12-10T18:53:51.965Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.942040278Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101102023-12-10T18:53:51.965Z{"msg":"[2]R flush_numbers[0..12]: [136, 135, 138, 138, 133, 133, 140, 135, 140, 135, 135, 135]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.942074455Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101112023-12-10T18:53:51.965Z{"msg":"[2]R generation[0..12]: [78, 78, 80, 80, 77, 77, 80, 78, 80, 78, 78, 78]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.94209672Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101122023-12-10T18:53:51.965Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.942118921Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101132023-12-10T18:53:51.965Z{"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.942160663Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101142023-12-10T18:53:51.965Z{"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.942183396Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101152023-12-10T18:53:51.965Z{"msg":"Next flush: 142","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.942203113Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101162023-12-10T18:53:51.966Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.942255142Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101172023-12-10T18:53:51.966Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.942277522Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101182023-12-10T18:53:51.966Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.942321194Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101192023-12-10T18:53:51.966Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
101202023-12-10T18:53:51.966Z,"time":"2023-12-10T18:53:51.942343384Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101212023-12-10T18:53:51.966Z{"msg":"0fdedeec-c218-485b-8d12-321532e9dfc0 is now active with session: f1f1555b-0857-4ea7-a501-e34e30f96649","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.942454441Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101222023-12-10T18:53:51.966Z{"msg":"0fdedeec-c218-485b-8d12-321532e9dfc0 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.942482325Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101232023-12-10T18:53:51.966Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.942524641Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101242023-12-10T18:53:51.966Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.942547214Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101252023-12-10T18:53:51.966Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.942588955Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101262023-12-10T18:53:51.966Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.942611908Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101272023-12-10T18:53:51.967Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.942631622Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101282023-12-10T18:53:51.967Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.942651371Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101292023-12-10T18:53:51.967Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.942691467Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101302023-12-10T18:53:51.967Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.942713286Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101312023-12-10T18:53:51.967Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.942752776Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101322023-12-10T18:53:51.967Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.9427753Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101332023-12-10T18:53:51.967Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:51.942815165Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1490,"session_id":"f1f1555b-0857-4ea7-a501-e34e30f96649"}
101342023-12-10T18:53:51.967ZLoading write count information from file "/tmp/test_repair_verify.data"
101352023-12-10T18:53:51.967ZRun Repair workload
101362023-12-10T18:53:51.967Z01/30 Write block 262 len 32768 data: 5 7 8 9 8 6 7 8
101372023-12-10T18:53:51.967Z02/30 Flush
101382023-12-10T18:53:51.988Z03/30 Write block 118 len 20480 data: 9 10 11 12 12
101392023-12-10T18:53:51.988Z04/30 Write block 566 len 8192 data: 10 9
101402023-12-10T18:53:51.988Z05/30 Write block 539 len 16384 data: 7 10 11 12
101412023-12-10T18:53:51.988Z06/30 Write block 241 len 28672 data: 13 13 12 11 10 11 9
101422023-12-10T18:53:51.988Z07/30 Flush
101432023-12-10T18:53:51.988Z08/30 Write block 109 len 32768 data: 7 7 6 6 7 5 7 9
101442023-12-10T18:53:51.988Z09/30 Write block 19 len 16384 data: 8 7 6 6
101452023-12-10T18:53:51.988Z10/30 Flush
101462023-12-10T18:53:51.988Z11/30 Write block 468 len 24576 data: 8 10 9 11 10 9
101472023-12-10T18:53:51.989Z12/30 Read block 450 len 12288
101482023-12-10T18:53:51.989Z13/30 Read block 230 len 4096
101492023-12-10T18:53:51.989Z14/30 Flush
101502023-12-10T18:53:51.989Z15/30 Write block 47 len 16384 data: 5 5 6 6
101512023-12-10T18:53:51.989Z16/30 Read block 578 len 8192
101522023-12-10T18:53:51.989Z17/30 Read block 133 len 8192
101532023-12-10T18:53:51.989Z18/30 Read block 293 len 4096
101542023-12-10T18:53:51.989Z19/30 Read block 532 len 24576
101552023-12-10T18:53:51.989Z20/30 Write block 480 len 28672 data: 2 2 3 4 4 4 6
101562023-12-10T18:53:51.989Z21/30 Write block 416 len 20480 data: 4 4 5 5 4
101572023-12-10T18:53:51.989Z22/30 Read block 393 len 24576
101582023-12-10T18:53:51.989Z23/30 Write block 197 len 28672 data: 13 11 10 9 9 7 9
101592023-12-10T18:53:51.989Z24/30 Read block 283 len 8192
101602023-12-10T18:53:51.989Z25/30 Write block 288 len 24576 data: 8 8 5 7 6 6
101612023-12-10T18:53:51.989Z26/30 Flush
101622023-12-10T18:53:51.989Z27/30 Write block 283 len 20480 data: 6 8 10 10 9
101632023-12-10T18:53:52.003Z28/30 Read block 491 len 32768
101642023-12-10T18:53:52.003Z29/30 Read block 575 len 12288
101652023-12-10T18:53:52.003Z30/30 Read block 382 len 16384
101662023-12-10T18:53:52.003Z----------------------------------------------------------------
101672023-12-10T18:53:52.003Z Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:28
101682023-12-10T18:53:52.004ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
101692023-12-10T18:53:52.004Z 3 Acked 1002 Write 5 Done Done Done false
101702023-12-10T18:53:52.004Z 4 Acked 1003 Write 2 Done Done Done false
101712023-12-10T18:53:52.005Z 5 Acked 1004 Write 4 Done Done Sent false
101722023-12-10T18:53:52.005Z 6 Acked 1005 Write 7 Done Done Sent false
101732023-12-10T18:53:52.005Z 7 Acked 1006 Flush 0 Done Done Sent false
101742023-12-10T18:53:52.005Z 8 Acked 1007 Write 8 Done Done Sent false
101752023-12-10T18:53:52.005Z 9 Acked 1008 Write 4 Done Done Sent false
101762023-12-10T18:53:52.005Z 10 Acked 1009 Flush 0 Done Done Sent false
101772023-12-10T18:53:52.005Z 11 Acked 1010 Write 6 Done Done Sent false
101782023-12-10T18:53:52.005Z 12 Acked 1011 Read 3 Done Done Sent false
101792023-12-10T18:53:52.005Z 13 Acked 1012 Read 1 Done Done Sent false
101802023-12-10T18:53:52.005Z 14 Acked 1013 Flush 0 Done Done Sent false
101812023-12-10T18:53:52.005Z 15 Acked 1014 Write 4 Done Done Sent false
101822023-12-10T18:53:52.005Z 16 Acked 1015 Read 2 Done Done Sent false
101832023-12-10T18:53:52.005Z 17 Acked 1016 Read 2 Done Done Sent false
101842023-12-10T18:53:52.005Z 18 Acked 1017 Read 1 Done Done Sent false
101852023-12-10T18:53:52.006Z 19 Acked 1018 Read 6 Done Done Sent false
101862023-12-10T18:53:52.006Z 20 Acked 1019 Write 7 Done Done Sent false
101872023-12-10T18:53:52.006Z 21 Acked 1020 Write 5 Done Done Sent false
101882023-12-10T18:53:52.006Z 22 Acked 1021 Read 6 Done Done Sent false
101892023-12-10T18:53:52.006Z 23 Acked 1022 Write 7 Done Done Sent false
101902023-12-10T18:53:52.006Z 24 Acked 1023 Read 2 Done Done Sent false
101912023-12-10T18:53:52.006Z 25 Acked 1024 Write 6 Done Done Sent false
101922023-12-10T18:53:52.006Z 26 Acked 1025 Flush 0 Done Done Sent false
101932023-12-10T18:53:52.006Z 27 Acked 1026 Write 5 Done Done Sent false
101942023-12-10T18:53:52.006Z 28 Acked 1027 Read 8 Done Done Sent false
101952023-12-10T18:53:52.006Z 29 Acked 1028 Read 3 Done Done Sent false
101962023-12-10T18:53:52.006Z 30 Acked 1029 Read 4 Done Done Sent false
101972023-12-10T18:53:52.006Z STATES DS:0 DS:1 DS:2 TOTAL
101982023-12-10T18:53:52.006Z New 0 0 0 0
101992023-12-10T18:53:52.006Z Sent 0 0 26 26
102002023-12-10T18:53:52.006Z Done 28 28 2 58
102012023-12-10T18:53:52.006Z Skipped 0 0 0 0
102022023-12-10T18:53:52.007Z Error 0 0 0 0
102032023-12-10T18:53:52.007ZLast Flush: 1025 1025 1001
102042023-12-10T18:53:52.007ZDownstairs last five completed: 1001 1000
102052023-12-10T18:53:52.007ZUpstairs last five completed: 30 29 28 27 26 25
102062023-12-10T18:53:52.007ZWrote out file "/tmp/test_repair_verify.data"
102072023-12-10T18:53:52.007Z
102082023-12-10T18:53:52.025ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
102092023-12-10T18:53:52.042ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
102102023-12-10T18:53:52.042Z 0 000-019 82 82 78 144 144 136 F F F <---
102112023-12-10T18:53:52.043Z 1 020-039 82 82 78 144 144 135 F F F <---
102122023-12-10T18:53:52.043Z 2 040-059 82 82 80 146 146 138 F F F <---
102132023-12-10T18:53:52.043Z 3 060-079 80 80 80 138 138 138 F F F
102142023-12-10T18:53:52.043Z 4 080-099 77 77 77 133 133 133 F F F
102152023-12-10T18:53:52.043Z 5 100-119 82 82 77 144 144 133 F F T <---
102162023-12-10T18:53:52.044Z 6 120-139 82 82 80 143 143 140 F F T <---
102172023-12-10T18:53:52.044Z 7 140-159 78 78 78 135 135 135 F F F
102182023-12-10T18:53:52.044Z 8 160-179 80 80 80 140 140 140 F F F
102192023-12-10T18:53:52.044Z 9 180-199 82 82 78 146 146 135 F F F <---
102202023-12-10T18:53:52.044Z 10 200-219 82 82 78 146 146 135 F F F <---
102212023-12-10T18:53:52.044Z 11 220-239 78 78 78 135 135 135 F F F
102222023-12-10T18:53:52.044Z 12 240-259 82 82 78 143 143 135 F F F <---
102232023-12-10T18:53:52.044Z 13 260-279 82 82 82 142 142 142 F F F
102242023-12-10T18:53:52.044Z 14 280-299 82 82 80 146 146 140 T T F <---
102252023-12-10T18:53:52.044Z 15 300-319 78 78 78 135 135 135 F F F
102262023-12-10T18:53:52.044Z 16 320-339 80 80 80 140 140 140 F F F
102272023-12-10T18:53:52.044Z 17 340-359 81 81 81 141 141 141 F F F
102282023-12-10T18:53:52.044Z 18 360-379 76 76 76 132 132 132 F F F
102292023-12-10T18:53:52.045Z 19 380-399 77 77 77 133 133 133 F F F
102302023-12-10T18:53:52.045Z 20 400-419 82 82 72 146 146 127 F F F <---
102312023-12-10T18:53:52.045Z 21 420-439 82 82 59 146 146 98 F F F <---
102322023-12-10T18:53:52.045Z 22 440-459 80 80 80 139 139 139 F F F
102332023-12-10T18:53:52.045Z 23 460-479 82 82 72 145 145 125 F F F <---
102342023-12-10T18:53:52.045Z 24 480-499 82 82 75 146 146 131 F F F <---
102352023-12-10T18:53:52.045Z 25 500-519 75 75 75 131 131 131 F F F
102362023-12-10T18:53:52.045Z 26 520-539 82 82 80 143 143 138 F F F <---
102372023-12-10T18:53:52.045Z 27 540-559 82 82 80 143 143 140 F F F <---
102382023-12-10T18:53:52.045Z 28 560-579 82 82 80 143 143 140 F F T <---
102392023-12-10T18:53:52.045Z 29 580-599 74 74 74 130 130 130 F F F
102402023-12-10T18:53:52.046ZMax gen: 82, Max flush: 146
102412023-12-10T18:53:52.046ZError: Difference in extent metadata found!
102422023-12-10T18:53:52.047ZA Difference in extent metadata is expected here
102432023-12-10T18:53:52.047ZOn loop 41
102442023-12-10T18:53:52.047Z
102452023-12-10T18:53:52.065ZVerifying data now
102462023-12-10T18:53:58.512ZNo change after verify
102472023-12-10T18:53:58.512ZLoop: 41 Downstairs dump after verify (and repair):
102482023-12-10T18:53:58.534ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
102492023-12-10T18:53:58.534Z 0 000-019 82 82 82 144 144 144 F F F
102502023-12-10T18:53:58.538Z 1 020-039 82 82 82 144 144 144 F F F
102512023-12-10T18:53:58.538Z 2 040-059 82 82 82 146 146 146 F F F
102522023-12-10T18:53:58.538Z 3 060-079 80 80 80 138 138 138 F F F
102532023-12-10T18:53:58.538Z 4 080-099 77 77 77 133 133 133 F F F
102542023-12-10T18:53:58.538Z 5 100-119 82 82 82 144 144 144 F F F
102552023-12-10T18:53:58.538Z 6 120-139 82 82 82 143 143 143 F F F
102562023-12-10T18:53:58.538Z 7 140-159 78 78 78 135 135 135 F F F
102572023-12-10T18:53:58.538Z 8 160-179 80 80 80 140 140 140 F F F
102582023-12-10T18:53:58.538Z 9 180-199 82 82 82 146 146 146 F F F
102592023-12-10T18:53:58.539Z 10 200-219 82 82 82 146 146 146 F F F
102602023-12-10T18:53:58.539Z 11 220-239 78 78 78 135 135 135 F F F
102612023-12-10T18:53:58.539Z 12 240-259 82 82 82 143 143 143 F F F
102622023-12-10T18:53:58.539Z 13 260-279 82 82 82 142 142 142 F F F
102632023-12-10T18:53:58.539Z 14 280-299 83 83 83 147 147 147 F F F
102642023-12-10T18:53:58.539Z 15 300-319 78 78 78 135 135 135 F F F
102652023-12-10T18:53:58.539Z 16 320-339 80 80 80 140 140 140 F F F
102662023-12-10T18:53:58.539Z 17 340-359 81 81 81 141 141 141 F F F
102672023-12-10T18:53:58.539Z 18 360-379 76 76 76 132 132 132 F F F
102682023-12-10T18:53:58.539Z 19 380-399 77 77 77 133 133 133 F F F
102692023-12-10T18:53:58.539Z 20 400-419 82 82 82 146 146 146 F F F
102702023-12-10T18:53:58.539Z 21 420-439 82 82 82 146 146 146 F F F
102712023-12-10T18:53:58.539Z 22 440-459 80 80 80 139 139 139 F F F
102722023-12-10T18:53:58.540Z 23 460-479 82 82 82 145 145 145 F F F
102732023-12-10T18:53:58.540Z 24 480-499 82 82 82 146 146 146 F F F
102742023-12-10T18:53:58.540Z 25 500-519 75 75 75 131 131 131 F F F
102752023-12-10T18:53:58.540Z 26 520-539 82 82 82 143 143 143 F F F
102762023-12-10T18:53:58.540Z 27 540-559 82 82 82 143 143 143 F F F
102772023-12-10T18:53:58.540Z 28 560-579 82 82 82 143 143 143 F F F
102782023-12-10T18:53:58.540Z 29 580-599 74 74 74 130 130 130 F F F
102792023-12-10T18:53:58.540ZMax gen: 83, Max flush: 147
102802023-12-10T18:53:58.540Z
102812023-12-10T18:53:58.540ZBegin loop 42
102822023-12-10T18:53:58.540ZDownstairs to restart: 1
102832023-12-10T18:53:58.566Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.570127671Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498}
102842023-12-10T18:53:58.570Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.570681279Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498}
102852023-12-10T18:53:58.570Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.570752617Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498}
102862023-12-10T18:53:58.570Z{"msg":"Crucible d0901c98-d381-44ba-8894-90cb4051a576 has session id: d7e64c50-84e2-49c4-b626-93958b51f105","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.570889148Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"Crucible runtime is spawned
102872023-12-10T18:53:58.570Z}
102882023-12-10T18:53:58.570Z{"msg":"The guest has requested activation with gen:84
102892023-12-10T18:53:58.570ZUpstairs opts: Upstairs UUID: d0901c98-d381-44ba-8894-90cb4051a576, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.571206322Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
102902023-12-10T18:53:58.570Z{"msg":"Crucible stats registered with UUID: d0901c98-d381-44ba-8894-90cb4051a576","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.571290557Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
102912023-12-10T18:53:58.571Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.571331758Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"looper":"0","session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
102922023-12-10T18:53:58.571Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.571400459Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105","task":"up_listen"}
102932023-12-10T18:53:58.571Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.571524622Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
102942023-12-10T18:53:58.572Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.57164294Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
102952023-12-10T18:53:58.572Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.571672751Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"looper":"2","session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
102962023-12-10T18:53:58.572Z{"msg":"[0] d0901c98-d381-44ba-8894-90cb4051a576 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.571706552Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"looper":"0","session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
102972023-12-10T18:53:58.572Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.571730201Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"looper":"1","session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
102982023-12-10T18:53:58.572Z{"msg":"[2] d0901c98-d381-44ba-8894-90cb4051a576 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.571753412Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"looper":"2","session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
102992023-12-10T18:53:58.572Z{"msg":"[1] d0901c98-d381-44ba-8894-90cb4051a576 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.571775321Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"looper":"1","session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103002023-12-10T18:53:58.572Z{"msg":"d0901c98-d381-44ba-8894-90cb4051a576 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.572000632Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103012023-12-10T18:53:58.572Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.572079266Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103022023-12-10T18:53:58.572Z{"msg":"Set desired generation to :84","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.572118391Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103032023-12-10T18:53:58.572Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.572406654Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103042023-12-10T18:53:58.572Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.572513036Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103052023-12-10T18:53:58.572Z{"msg":"[0] d0901c98-d381-44ba-8894-90cb4051a576 (d7e64c50-84e2-49c4-b626-93958b51f105) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.572773725Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103062023-12-10T18:53:58.572Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.572881187Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103072023-12-10T18:53:58.572Z{"msg":"[0] client is_active_req TRUE, promote! session d7e64c50-84e2-49c4-b626-93958b51f105","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.57292643Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103082023-12-10T18:53:58.572Z{"msg":"[2] d0901c98-d381-44ba-8894-90cb4051a576 (d7e64c50-84e2-49c4-b626-93958b51f105) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.573108623Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103092023-12-10T18:53:58.572Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.573208838Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103102023-12-10T18:53:58.573Z{"msg":"[2] client is_active_req TRUE, promote! session d7e64c50-84e2-49c4-b626-93958b51f105","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.573360278Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103112023-12-10T18:53:58.573Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.573477126Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103122023-12-10T18:53:58.573Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.573560112Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103132023-12-10T18:53:58.573Z{"msg":"d0901c98-d381-44ba-8894-90cb4051a576 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.573638946Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103142023-12-10T18:53:58.573Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.573682033Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103152023-12-10T18:53:58.573Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.573779764Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103162023-12-10T18:53:58.577Z{"msg":"d0901c98-d381-44ba-8894-90cb4051a576 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.573820309Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103172023-12-10T18:53:58.578Z{"msg":"[1] d0901c98-d381-44ba-8894-90cb4051a576 (d7e64c50-84e2-49c4-b626-93958b51f105) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.574019937Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103182023-12-10T18:53:58.578Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.574095152Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103192023-12-10T18:53:58.578Z{"msg":"[1] client is_active_req TRUE, promote! session d7e64c50-84e2-49c4-b626-93958b51f105","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.57413484Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103202023-12-10T18:53:58.578Z{"msg":"[0] d0901c98-d381-44ba-8894-90cb4051a576 (d7e64c50-84e2-49c4-b626-93958b51f105) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.574469398Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103212023-12-10T18:53:58.578Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.574540277Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103222023-12-10T18:53:58.578Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:58.574564612Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103232023-12-10T18:53:58.578Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.574663737Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103242023-12-10T18:53:58.578Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.574725482Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103252023-12-10T18:53:58.578Z{"msg":"d0901c98-d381-44ba-8894-90cb4051a576 WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.574774429Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103262023-12-10T18:53:58.578Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.57479827Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103272023-12-10T18:53:58.578Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.57482033Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103282023-12-10T18:53:58.578Z{"msg":"[2] d0901c98-d381-44ba-8894-90cb4051a576 (d7e64c50-84e2-49c4-b626-93958b51f105) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.575008582Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103292023-12-10T18:53:58.578Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.575160259Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103302023-12-10T18:53:58.578Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:58.575188433Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103312023-12-10T18:53:58.578Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.57523125Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103322023-12-10T18:53:58.578Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.575254741Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103332023-12-10T18:53:58.578Z{"msg":"d0901c98-d381-44ba-8894-90cb4051a576 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.575276316Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103342023-12-10T18:53:58.579Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.575316684Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103352023-12-10T18:53:58.579Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.575338661Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103362023-12-10T18:53:58.580Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.575379979Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103372023-12-10T18:53:58.580Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.575407244Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103382023-12-10T18:53:58.580Z{"msg":"d0901c98-d381-44ba-8894-90cb4051a576 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.575448465Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103392023-12-10T18:53:58.580Z{"msg":"[1] d0901c98-d381-44ba-8894-90cb4051a576 (d7e64c50-84e2-49c4-b626-93958b51f105) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.575894726Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103402023-12-10T18:53:58.580Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.575966795Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103412023-12-10T18:53:58.580Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:53:58.575990765Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103422023-12-10T18:53:58.580Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.576010933Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103432023-12-10T18:53:58.580Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.576051558Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103442023-12-10T18:53:58.580Z{"msg":"d0901c98-d381-44ba-8894-90cb4051a576 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.576074945Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103452023-12-10T18:53:58.580Z{"msg":"[0]R flush_numbers[0..12]: [144, 144, 146, 138, 133, 144, 143, 135, 140, 146, 146, 135]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.576116574Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103462023-12-10T18:53:58.580Z{"msg":"[0]R generation[0..12]: [82, 82, 82, 80, 77, 82, 82, 78, 80, 82, 82, 78]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.576140143Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103472023-12-10T18:53:58.580Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.576181935Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103482023-12-10T18:53:58.580Z{"msg":"[1]R flush_numbers[0..12]: [144, 144, 146, 138, 133, 144, 143, 135, 140, 146, 146, 135]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.576205778Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103492023-12-10T18:53:58.580ZThe guest has finished waiting for activation with:84
103502023-12-10T18:53:58.580Z{"msg":"[1]R generation[0..12]: [82, 82, 82, 80, 77, 82, 82, 78, 80, 82, 82, 78]",Wait for a query_work_queue command to finish before sending IO
103512023-12-10T18:53:58.580Z"v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.576252401Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103522023-12-10T18:53:58.581Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.576276745Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103532023-12-10T18:53:58.581Z{"msg":"[2]R flush_numbers[0..12]: [144, 144, 146, 138, 133, 144, 143, 135, 140, 146, 146, 135]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.576318325Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103542023-12-10T18:53:58.581Z{"msg":"[2]R generation[0..12]: [82, 82, 82, 80, 77, 82, 82, 78, 80, 82, 82, 78]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.576341417Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103552023-12-10T18:53:58.581Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.576377609Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103562023-12-10T18:53:58.581Z{"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.576422839Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103572023-12-10T18:53:58.581Z{"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.57646158Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103582023-12-10T18:53:58.581Z{"msg":"Next flush: 148","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.576531588Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103592023-12-10T18:53:58.581Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.576599227Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103602023-12-10T18:53:58.581Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.576648269Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103612023-12-10T18:53:58.581Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.576671025Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103622023-12-10T18:53:58.581Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.576690757Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103632023-12-10T18:53:58.581Z{"msg":"d0901c98-d381-44ba-8894-90cb4051a576 is now active with session: d7e64c50-84e2-49c4-b626-93958b51f105","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.57673544Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103642023-12-10T18:53:58.581Z{"msg":"d0901c98-d381-44ba-8894-90cb4051a576 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.576758207Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103652023-12-10T18:53:58.581Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.576778568Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103662023-12-10T18:53:58.581Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.576830428Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103672023-12-10T18:53:58.581Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.576860469Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103682023-12-10T18:53:58.581Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.576903567Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103692023-12-10T18:53:58.582Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.576925786Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103702023-12-10T18:53:58.582Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.576988228Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103712023-12-10T18:53:58.582Z{Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
103722023-12-10T18:53:58.582Z"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.577066289Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103732023-12-10T18:53:58.582Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.577113331Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103742023-12-10T18:53:58.582Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.577135949Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103752023-12-10T18:53:58.582Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.577155682Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103762023-12-10T18:53:58.582Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:53:58.577194747Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1498,"session_id":"d7e64c50-84e2-49c4-b626-93958b51f105"}
103772023-12-10T18:53:58.582ZLoading write count information from file "/tmp/test_repair_verify.data"
103782023-12-10T18:53:58.582ZRun Repair workload
103792023-12-10T18:53:58.582Z01/30 Write block 506 len 40960 data: 9 8 8 7 8 8 7 8 7 6
103802023-12-10T18:53:58.582Z02/30 Read block 368 len 28672
103812023-12-10T18:53:58.588Z03/30 Write block 10 len 4096 data: 6
103822023-12-10T18:53:58.588Z04/30 Read block 278 len 32768
103832023-12-10T18:53:58.588Z05/30 Write block 373 len 40960 data: 6 7 10 10 9 12 11 12 8 9
103842023-12-10T18:53:58.588Z06/30 Read block 344 len 24576
103852023-12-10T18:53:58.592Z07/30 Read block 306 len 40960
103862023-12-10T18:53:58.592Z08/30 Write block 579 len 16384 data: 7 7 8 7
103872023-12-10T18:53:58.592Z09/30 Read block 98 len 36864
103882023-12-10T18:53:58.595Z10/30 Read block 339 len 28672
103892023-12-10T18:53:58.597Z11/30 Flush
103902023-12-10T18:53:58.600Z12/30 Write block 88 len 40960 data: 9 7 6 5 4 6 6 5 4 4
103912023-12-10T18:53:58.603Z13/30 Write block 227 len 16384 data: 9 8 8 9
103922023-12-10T18:53:58.603Z14/30 Read block 349 len 12288
103932023-12-10T18:53:58.603Z15/30 Read block 219 len 28672
103942023-12-10T18:53:58.606Z16/30 Write block 287 len 24576 data: 10 9 9 6 8 7
103952023-12-10T18:53:58.609Z17/30 Read block 567 len 12288
103962023-12-10T18:53:58.609Z18/30 Read block 106 len 32768
103972023-12-10T18:53:58.611Z19/30 Read block 423 len 28672
103982023-12-10T18:53:58.614Z20/30 Write block 168 len 20480 data: 9 8 7 7 7
103992023-12-10T18:53:58.614Z21/30 Read block 70 len 32768
104002023-12-10T18:53:58.618Z22/30 Read block 586 len 4096
104012023-12-10T18:53:58.621Z23/30 Write block 162 len 28672 data: 8 9 9 10 12 12 10
104022023-12-10T18:53:58.622Z24/30 Write block 335 len 28672 data: 6 6 7 6 7 8 9
104032023-12-10T18:53:58.622Z25/30 Write block 435 len 24576 data: 4 3 4 5 5 7
104042023-12-10T18:53:58.624Z26/30 Read block 91 len 40960
104052023-12-10T18:53:58.627Z27/30 Read block 29 len 4096
104062023-12-10T18:53:58.627Z28/30 Write block 10 len 40960 data: 7 6 8 11 12 11 9 10 8 9
104072023-12-10T18:53:58.631Z29/30 Write block 496 len 4096 data: 5
104082023-12-10T18:53:58.631Z30/30 Write block 102 len 28672 data: 9 8 5 5 6 6 8
104092023-12-10T18:53:58.631Z----------------------------------------------------------------
104102023-12-10T18:53:58.631Z Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30
104112023-12-10T18:53:58.632ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
104122023-12-10T18:53:58.632Z 1 Acked 1000 Write 10 Done Done Done false
104132023-12-10T18:53:58.632Z 2 Acked 1001 Read 7 Done Sent Done false
104142023-12-10T18:53:58.632Z 3 Acked 1002 Write 1 Done Sent Done false
104152023-12-10T18:53:58.632Z 4 Acked 1003 Read 8 Done Sent Done false
104162023-12-10T18:53:58.632Z 5 Acked 1004 Write 10 Done Sent Done false
104172023-12-10T18:53:58.632Z 6 Acked 1005 Read 6 Done Sent Done false
104182023-12-10T18:53:58.632Z 7 Acked 1006 Read 10 Done Sent Done false
104192023-12-10T18:53:58.632Z 8 Acked 1007 Write 4 Done Sent Done false
104202023-12-10T18:53:58.632Z 9 Acked 1008 Read 9 Done Sent Done false
104212023-12-10T18:53:58.632Z 10 Acked 1009 Read 7 Done Sent Done false
104222023-12-10T18:53:58.632Z 11 Acked 1010 Flush 0 Done Sent Done false
104232023-12-10T18:53:58.632Z 12 Acked 1011 Write 10 Done Sent Done false
104242023-12-10T18:53:58.632Z 13 Acked 1012 Write 4 Done Sent Done false
104252023-12-10T18:53:58.632Z 14 Acked 1013 Read 3 Done Sent Done false
104262023-12-10T18:53:58.632Z 15 Acked 1014 Read 7 Done Sent Done false
104272023-12-10T18:53:58.632Z 16 Acked 1015 Write 6 Done Sent Done false
104282023-12-10T18:53:58.633Z 17 Acked 1016 Read 3 Done Sent Done false
104292023-12-10T18:53:58.633Z 18 Acked 1017 Read 8 Done Sent Done false
104302023-12-10T18:53:58.633Z 19 Acked 1018 Read 7 Done Sent Done false
104312023-12-10T18:53:58.633Z 20 Acked 1019 Write 5 Done Sent Done false
104322023-12-10T18:53:58.633Z 21 Acked 1020 Read 8 Done Sent Done false
104332023-12-10T18:53:58.633Z 22 Acked 1021 Read 1 Done Sent Done false
104342023-12-10T18:53:58.633Z 23 Acked 1022 Write 7 Done Sent Done false
104352023-12-10T18:53:58.633Z 24 Acked 1023 Write 7 Done Sent Done false
104362023-12-10T18:53:58.633Z 25 Acked 1024 Write 6 Done Sent Done false
104372023-12-10T18:53:58.633Z 26 Acked 1025 Read 10 Done Sent Done false
104382023-12-10T18:53:58.633Z 27 Acked 1026 Read 1 Done Sent Done false
104392023-12-10T18:53:58.633Z 28 Acked 1027 Write 10 Done Sent Done false
104402023-12-10T18:53:58.633Z 29 Acked 1028 Write 1 Done Sent Done false
104412023-12-10T18:53:58.633Z 30 Acked 1029 Write 7 Sent Sent Sent false
104422023-12-10T18:53:58.633Z STATES DS:0 DS:1 DS:2 TOTAL
104432023-12-10T18:53:58.633Z New 0 0 0 0
104442023-12-10T18:53:58.635Z Sent 1 29 1 31
104452023-12-10T18:53:58.635Z Done 29 1 29 59
104462023-12-10T18:53:58.635Z Skipped 0 0 0 0
104472023-12-10T18:53:58.635Z Error 0 0 0 0
104482023-12-10T18:53:58.635ZLast Flush: 1010 0 1010
104492023-12-10T18:53:58.635ZDownstairs last five completed:
104502023-12-10T18:53:58.635ZUpstairs last five completed: 30 29 28 27 26 25
104512023-12-10T18:53:58.635ZWrote out file "/tmp/test_repair_verify.data"
104522023-12-10T18:53:58.639Z
104532023-12-10T18:53:58.642ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
104542023-12-10T18:53:58.671ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
104552023-12-10T18:53:58.671Z 0 000-019 84 82 84 148 144 148 T F T <---
104562023-12-10T18:53:58.674Z 1 020-039 82 82 82 144 144 144 F F F
104572023-12-10T18:53:58.675Z 2 040-059 82 82 82 146 146 146 F F F
104582023-12-10T18:53:58.675Z 3 060-079 80 80 80 138 138 138 F F F
104592023-12-10T18:53:58.675Z 4 080-099 77 77 77 133 133 133 T F T <---
104602023-12-10T18:53:58.675Z 5 100-119 82 82 82 144 144 144 T F T <---
104612023-12-10T18:53:58.675Z 6 120-139 82 82 82 143 143 143 F F F
104622023-12-10T18:53:58.675Z 7 140-159 78 78 78 135 135 135 F F F
104632023-12-10T18:53:58.675Z 8 160-179 80 80 80 140 140 140 T F T <---
104642023-12-10T18:53:58.675Z 9 180-199 82 82 82 146 146 146 F F F
104652023-12-10T18:53:58.675Z 10 200-219 82 82 82 146 146 146 F F F
104662023-12-10T18:53:58.675Z 11 220-239 78 78 78 135 135 135 T F T <---
104672023-12-10T18:53:58.675Z 12 240-259 82 82 82 143 143 143 F F F
104682023-12-10T18:53:58.675Z 13 260-279 82 82 82 142 142 142 F F F
104692023-12-10T18:53:58.675Z 14 280-299 83 83 83 147 147 147 T F T <---
104702023-12-10T18:53:58.675Z 15 300-319 78 78 78 135 135 135 F F F
104712023-12-10T18:53:58.675Z 16 320-339 80 80 80 140 140 140 T F T <---
104722023-12-10T18:53:58.675Z 17 340-359 81 81 81 141 141 141 T F T <---
104732023-12-10T18:53:58.675Z 18 360-379 84 76 84 148 132 148 F F F <---
104742023-12-10T18:53:58.676Z 19 380-399 84 77 84 148 133 148 F F F <---
104752023-12-10T18:53:58.676Z 20 400-419 82 82 82 146 146 146 F F F
104762023-12-10T18:53:58.676Z 21 420-439 82 82 82 146 146 146 T F T <---
104772023-12-10T18:53:58.676Z 22 440-459 80 80 80 139 139 139 T F T <---
104782023-12-10T18:53:58.676Z 23 460-479 82 82 82 145 145 145 F F F
104792023-12-10T18:53:58.676Z 24 480-499 82 82 82 146 146 146 T F T <---
104802023-12-10T18:53:58.677Z 25 500-519 84 75 84 148 131 148 F T F <---
104812023-12-10T18:53:58.677Z 26 520-539 82 82 82 143 143 143 F F F
104822023-12-10T18:53:58.677Z 27 540-559 82 82 82 143 143 143 F F F
104832023-12-10T18:53:58.677Z 28 560-579 84 82 84 148 143 148 F F F <---
104842023-12-10T18:53:58.677Z 29 580-599 84 74 84 148 130 148 F F F <---
104852023-12-10T18:53:58.678ZMax gen: 84, Max flush: 148
104862023-12-10T18:53:58.678ZError: Difference in extent metadata found!
104872023-12-10T18:53:58.683ZA Difference in extent metadata is expected here
104882023-12-10T18:53:58.684ZOn loop 42
104892023-12-10T18:53:58.684Z
104902023-12-10T18:53:58.684ZVerifying data now
104912023-12-10T18:54:04.209ZNo change after verify
104922023-12-10T18:54:04.210ZLoop: 42 Downstairs dump after verify (and repair):
104932023-12-10T18:54:04.253ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
104942023-12-10T18:54:04.253Z 0 000-019 85 85 85 149 149 149 F F F
104952023-12-10T18:54:04.257Z 1 020-039 82 82 82 144 144 144 F F F
104962023-12-10T18:54:04.257Z 2 040-059 82 82 82 146 146 146 F F F
104972023-12-10T18:54:04.257Z 3 060-079 80 80 80 138 138 138 F F F
104982023-12-10T18:54:04.257Z 4 080-099 85 85 85 149 149 149 F F F
104992023-12-10T18:54:04.257Z 5 100-119 85 85 85 149 149 149 F F F
105002023-12-10T18:54:04.258Z 6 120-139 82 82 82 143 143 143 F F F
105012023-12-10T18:54:04.258Z 7 140-159 78 78 78 135 135 135 F F F
105022023-12-10T18:54:04.258Z 8 160-179 85 85 85 149 149 149 F F F
105032023-12-10T18:54:04.258Z 9 180-199 82 82 82 146 146 146 F F F
105042023-12-10T18:54:04.258Z 10 200-219 82 82 82 146 146 146 F F F
105052023-12-10T18:54:04.258Z 11 220-239 85 85 85 149 149 149 F F F
105062023-12-10T18:54:04.258Z 12 240-259 82 82 82 143 143 143 F F F
105072023-12-10T18:54:04.258Z 13 260-279 82 82 82 142 142 142 F F F
105082023-12-10T18:54:04.258Z 14 280-299 85 85 85 149 149 149 F F F
105092023-12-10T18:54:04.258Z 15 300-319 78 78 78 135 135 135 F F F
105102023-12-10T18:54:04.258Z 16 320-339 85 85 85 149 149 149 F F F
105112023-12-10T18:54:04.258Z 17 340-359 85 85 85 149 149 149 F F F
105122023-12-10T18:54:04.258Z 18 360-379 84 84 84 148 148 148 F F F
105132023-12-10T18:54:04.258Z 19 380-399 84 84 84 148 148 148 F F F
105142023-12-10T18:54:04.258Z 20 400-419 82 82 82 146 146 146 F F F
105152023-12-10T18:54:04.258Z 21 420-439 85 85 85 149 149 149 F F F
105162023-12-10T18:54:04.258Z 22 440-459 85 85 85 149 149 149 F F F
105172023-12-10T18:54:04.259Z 23 460-479 82 82 82 145 145 145 F F F
105182023-12-10T18:54:04.259Z 24 480-499 85 85 85 149 149 149 F F F
105192023-12-10T18:54:04.259Z 25 500-519 84 84 84 148 148 148 F F F
105202023-12-10T18:54:04.259Z 26 520-539 82 82 82 143 143 143 F F F
105212023-12-10T18:54:04.259Z 27 540-559 82 82 82 143 143 143 F F F
105222023-12-10T18:54:04.259Z 28 560-579 84 84 84 148 148 148 F F F
105232023-12-10T18:54:04.259Z 29 580-599 84 84 84 148 148 148 F F F
105242023-12-10T18:54:04.259ZMax gen: 85, Max flush: 149
105252023-12-10T18:54:04.260Z
105262023-12-10T18:54:04.260ZBegin loop 43
105272023-12-10T18:54:04.260ZDownstairs to restart: 2
105282023-12-10T18:54:04.260Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.259711321Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506}
105292023-12-10T18:54:04.260Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.260141233Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506}
105302023-12-10T18:54:04.260Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.260171186Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506}
105312023-12-10T18:54:04.260Z{"msg":"Crucible 97193113-de4b-4808-98ab-698e73a20940 has session id: 14573a40-6c7f-417a-a606-de242d58bb4d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.260216171Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105322023-12-10T18:54:04.260Z{"msg":"Upstairs opts: Upstairs UUID: 97193113-de4b-4808-98ab-698e73a20940, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.26025453Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105332023-12-10T18:54:04.260Z{"msg":"Crucible stats registered with UUID: 97193113-de4b-4808-98ab-698e73a20940","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.260279297Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105342023-12-10T18:54:04.260ZCrucible runtime is spawned
105352023-12-10T18:54:04.260ZThe guest has requested activation with gen:86
105362023-12-10T18:54:04.260Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.260577192Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d","task":"up_listen"}
105372023-12-10T18:54:04.260Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.260662726Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"looper":"1","session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105382023-12-10T18:54:04.260Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.260744708Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105392023-12-10T18:54:04.260Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.260786835Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105402023-12-10T18:54:04.266Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.261009203Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"looper":"0","session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105412023-12-10T18:54:04.266Z{"msg":"[1] 97193113-de4b-4808-98ab-698e73a20940 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.261064178Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"looper":"1","session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105422023-12-10T18:54:04.266Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.261116888Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"looper":"2","session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105432023-12-10T18:54:04.267Z{"msg":"[0] 97193113-de4b-4808-98ab-698e73a20940 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.261195247Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"looper":"0","session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105442023-12-10T18:54:04.268Z{"msg":"[2] 97193113-de4b-4808-98ab-698e73a20940 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.261404459Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"looper":"2","session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105452023-12-10T18:54:04.268Z{"msg":"97193113-de4b-4808-98ab-698e73a20940 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.261647407Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105462023-12-10T18:54:04.268Z{"msg":"Set desired generation to :86","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.261741557Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105472023-12-10T18:54:04.268Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.261914418Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105482023-12-10T18:54:04.268Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.262062448Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105492023-12-10T18:54:04.269Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.262319601Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105502023-12-10T18:54:04.269Z{"msg":"[1] 97193113-de4b-4808-98ab-698e73a20940 (14573a40-6c7f-417a-a606-de242d58bb4d) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.262780637Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105512023-12-10T18:54:04.269Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.262868115Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105522023-12-10T18:54:04.269Z{"msg":"[1] client is_active_req TRUE, promote! session 14573a40-6c7f-417a-a606-de242d58bb4d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.262956825Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105532023-12-10T18:54:04.269Z{"msg":"[0] 97193113-de4b-4808-98ab-698e73a20940 (14573a40-6c7f-417a-a606-de242d58bb4d) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.263004474Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105542023-12-10T18:54:04.269Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.263040981Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105552023-12-10T18:54:04.269Z{"msg":"[0] client is_active_req TRUE, promote! session 14573a40-6c7f-417a-a606-de242d58bb4d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.263133278Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105562023-12-10T18:54:04.269Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.263535651Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105572023-12-10T18:54:04.269Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.263578373Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105582023-12-10T18:54:04.269Z{"msg":"97193113-de4b-4808-98ab-698e73a20940 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.263602057Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105592023-12-10T18:54:04.270Z{"msg":"[2] 97193113-de4b-4808-98ab-698e73a20940 (14573a40-6c7f-417a-a606-de242d58bb4d) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.263722755Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105602023-12-10T18:54:04.270Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.263831303Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105612023-12-10T18:54:04.270Z{"msg":"[2] client is_active_req TRUE, promote! session 14573a40-6c7f-417a-a606-de242d58bb4d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.263915973Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105622023-12-10T18:54:04.270Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.263964278Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105632023-12-10T18:54:04.271Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.264043037Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105642023-12-10T18:54:04.271Z{"msg":"97193113-de4b-4808-98ab-698e73a20940 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.264181985Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105652023-12-10T18:54:04.271Z{"msg":"[1] 97193113-de4b-4808-98ab-698e73a20940 (14573a40-6c7f-417a-a606-de242d58bb4d) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.264310735Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105662023-12-10T18:54:04.271Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.26439778Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105672023-12-10T18:54:04.271Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:04.264440019Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105682023-12-10T18:54:04.271Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.264519914Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105692023-12-10T18:54:04.271Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.264562519Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105702023-12-10T18:54:04.271Z{"msg":"97193113-de4b-4808-98ab-698e73a20940 WaitActive WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.26460106Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105712023-12-10T18:54:04.272Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.264637142Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105722023-12-10T18:54:04.272Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.264675564Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105732023-12-10T18:54:04.272Z{"msg":"[0] 97193113-de4b-4808-98ab-698e73a20940 (14573a40-6c7f-417a-a606-de242d58bb4d) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.264753458Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105742023-12-10T18:54:04.273Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.26481565Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105752023-12-10T18:54:04.273Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:04.264841406Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105762023-12-10T18:54:04.273Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.264895797Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105772023-12-10T18:54:04.273Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.264969839Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105782023-12-10T18:54:04.273Z{"msg":"97193113-de4b-4808-98ab-698e73a20940 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.265045979Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105792023-12-10T18:54:04.273Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.265116233Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105802023-12-10T18:54:04.273Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.265185238Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105812023-12-10T18:54:04.273Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.26525644Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105822023-12-10T18:54:04.274Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.26530429Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105832023-12-10T18:54:04.275Z{"msg":"97193113-de4b-4808-98ab-698e73a20940 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.265374541Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105842023-12-10T18:54:04.275Z{"msg":"[2] 97193113-de4b-4808-98ab-698e73a20940 (14573a40-6c7f-417a-a606-de242d58bb4d) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.265685611Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105852023-12-10T18:54:04.275Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.265763202Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105862023-12-10T18:54:04.276Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:04.265788582Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105872023-12-10T18:54:04.276Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.265811111Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105882023-12-10T18:54:04.276Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.265852985Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105892023-12-10T18:54:04.276Z{"msg":"97193113-de4b-4808-98ab-698e73a20940 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.265876912Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105902023-12-10T18:54:04.276Z{"msg":"[0]R flush_numbers[0..12]: [149, 144, 146, 138, 149, 149, 143, 135, 149, 146, 146, 149]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.265918964Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105912023-12-10T18:54:04.276Z{"msg":"[0]R generation[0..12]: [85, 82, 82, 80, 85, 85, 82, 78, 85, 82, 82, 85]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.265942516Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105922023-12-10T18:54:04.276Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.265964512Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105932023-12-10T18:54:04.276Z{"msg":"[1]R flush_numbers[0..12]: [149, 144, 146, 138, 149, 149, 143, 135, 149, 146, 146, 149]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:86
105942023-12-10T18:54:04.276ZWait for a query_work_queue command to finish before sending IO
105952023-12-10T18:54:04.277Z,"time":"2023-12-10T18:54:04.265986009Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105962023-12-10T18:54:04.277Z{"msg":"[1]R generation[0..12]: [85, 82, 82, 80, 85, 85, 82, 78, 85, 82, 82, 85]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.266127855Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105972023-12-10T18:54:04.277Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.266176371Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105982023-12-10T18:54:04.277Z{"msg":"[2]R flush_numbers[0..12]: [149, 144, 146, 138, 149, 149, 143, 135, 149, 146, 146, 149]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.266284673Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
105992023-12-10T18:54:04.277Z{"msg":"[2]R generation[0..12]: [85, 82, 82, 80, 85, 85, 82, 78, 85, 82, 82, 85]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.26633057Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
106002023-12-10T18:54:04.277Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.266363519Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
106012023-12-10T18:54:04.277Z{"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.266414011Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
106022023-12-10T18:54:04.277Z{"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.266437923Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
106032023-12-10T18:54:04.277Z{"msg":"Next flush: 150","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.266485707Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
106042023-12-10T18:54:04.278Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.266507887Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
106052023-12-10T18:54:04.278Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.266529683Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
106062023-12-10T18:54:04.278Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.266550235Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
106072023-12-10T18:54:04.278Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.266569998Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
106082023-12-10T18:54:04.278Z{"msg":"97193113-de4b-4808-98ab-698e73a20940 is now active with session: 14573a40-6c7f-417a-a606-de242d58bb4d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.266613122Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
106092023-12-10T18:54:04.278Z{"msg":"97193113-de4b-4808-98ab-698e73a20940 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.266636353Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
106102023-12-10T18:54:04.278Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.266677475Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
106112023-12-10T18:54:04.278Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.266699536Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
106122023-12-10T18:54:04.278ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
106132023-12-10T18:54:04.278Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.266804244Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
106142023-12-10T18:54:04.279Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.266854545Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
106152023-12-10T18:54:04.279Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.26687797Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
106162023-12-10T18:54:04.279Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.266918906Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
106172023-12-10T18:54:04.279Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.266941481Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
106182023-12-10T18:54:04.279Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.266981827Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
106192023-12-10T18:54:04.279Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.26700417Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
106202023-12-10T18:54:04.279Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.267024322Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
106212023-12-10T18:54:04.279Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:04.267064163Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1506,"session_id":"14573a40-6c7f-417a-a606-de242d58bb4d"}
106222023-12-10T18:54:04.279ZLoading write count information from file "/tmp/test_repair_verify.data"
106232023-12-10T18:54:04.279ZRun Repair workload
106242023-12-10T18:54:04.280Z01/30 Write block 230 len 12288 data: 10 9 9
106252023-12-10T18:54:04.334Z02/30 Read block 468 len 36864
106262023-12-10T18:54:04.334Z03/30 Read block 567 len 40960
106272023-12-10T18:54:04.334Z04/30 Write block 149 len 8192 data: 7 6
106282023-12-10T18:54:04.334Z05/30 Write block 275 len 36864 data: 10 9 9 10 7 6 5 6 7
106292023-12-10T18:54:04.334Z06/30 Read block 371 len 32768
106302023-12-10T18:54:04.334Z07/30 Read block 215 len 40960
106312023-12-10T18:54:04.334Z08/30 Read block 463 len 20480
106322023-12-10T18:54:04.334Z09/30 Read block 281 len 8192
106332023-12-10T18:54:04.335Z10/30 Read block 382 len 4096
106342023-12-10T18:54:04.335Z11/30 Read block 409 len 4096
106352023-12-10T18:54:04.340Z12/30 Write block 378 len 36864 data: 13 12 13 9 10 9 10 10 9
106362023-12-10T18:54:04.340Z13/30 Write block 162 len 16384 data: 9 10 10 11
106372023-12-10T18:54:04.340Z14/30 Write block 170 len 4096 data: 8
106382023-12-10T18:54:04.340Z15/30 Read block 241 len 40960
106392023-12-10T18:54:04.340Z16/30 Write block 198 len 32768 data: 12 11 10 10 8 10 8 8
106402023-12-10T18:54:04.340Z17/30 Read block 549 len 28672
106412023-12-10T18:54:04.340Z18/30 Write block 451 len 24576 data: 8 8 8 6 7 7
106422023-12-10T18:54:04.340Z19/30 Read block 214 len 16384
106432023-12-10T18:54:04.340Z20/30 Write block 319 len 36864 data: 5 6 8 10 9 10 8 8 8
106442023-12-10T18:54:04.340Z21/30 Read block 32 len 16384
106452023-12-10T18:54:04.340Z22/30 Read block 475 len 24576
106462023-12-10T18:54:04.340Z23/30 Write block 418 len 40960 data: 6 6 5 6 6 6 7 5 4 5
106472023-12-10T18:54:04.341Z24/30 Write block 489 len 24576 data: 10 9 10 6 4 5
106482023-12-10T18:54:04.341Z25/30 Read block 257 len 20480
106492023-12-10T18:54:04.341Z26/30 Write block 171 len 12288 data: 8 8 7
106502023-12-10T18:54:04.341Z27/30 Read block 420 len 8192
106512023-12-10T18:54:04.341Z28/30 Read block 258 len 8192
106522023-12-10T18:54:04.341Z29/30 Read block 519 len 12288
106532023-12-10T18:54:04.341Z30/30 Read block 178 len 32768
106542023-12-10T18:54:04.341Z----------------------------------------------------------------
106552023-12-10T18:54:04.341Z Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30
106562023-12-10T18:54:04.342ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
106572023-12-10T18:54:04.342Z 1 Acked 1000 Write 3 Done Done Done false
106582023-12-10T18:54:04.342Z 2 Acked 1001 Read 9 Done Done Done false
106592023-12-10T18:54:04.342Z 3 Acked 1002 Read 10 Done Done Done false
106602023-12-10T18:54:04.342Z 4 Acked 1003 Write 2 Done Done Done false
106612023-12-10T18:54:04.342Z 5 Acked 1004 Write 9 Done Done Done false
106622023-12-10T18:54:04.342Z 6 Acked 1005 Read 8 Done Done Done false
106632023-12-10T18:54:04.342Z 7 Acked 1006 Read 10 Done Done Done false
106642023-12-10T18:54:04.342Z 8 Acked 1007 Read 5 Done Done Done false
106652023-12-10T18:54:04.342Z 9 Acked 1008 Read 2 Done Done Done false
106662023-12-10T18:54:04.342Z 10 Acked 1009 Read 1 Done Done Done false
106672023-12-10T18:54:04.343Z 11 Acked 1010 Read 1 Done Done Done false
106682023-12-10T18:54:04.343Z 12 Acked 1011 Write 9 Done Done Done false
106692023-12-10T18:54:04.343Z 13 Acked 1012 Write 4 Done Done Done false
106702023-12-10T18:54:04.343Z 14 Acked 1013 Write 1 Done Done Done false
106712023-12-10T18:54:04.343Z 15 Acked 1014 Read 10 Done Done Done false
106722023-12-10T18:54:04.343Z 16 Acked 1015 Write 8 Done Done Done false
106732023-12-10T18:54:04.343Z 17 Acked 1016 Read 7 Done Done Done false
106742023-12-10T18:54:04.343Z 18 Acked 1017 Write 6 Done Done Sent false
106752023-12-10T18:54:04.343Z 19 Acked 1018 Read 4 Done Done Sent false
106762023-12-10T18:54:04.343Z 20 Acked 1019 Write 9 Done Done Sent false
106772023-12-10T18:54:04.344Z 21 Acked 1020 Read 4 Done Done Sent false
106782023-12-10T18:54:04.344Z 22 Acked 1021 Read 6 Done Done Sent false
106792023-12-10T18:54:04.347Z 23 Acked 1022 Write 10 Done Done Sent false
106802023-12-10T18:54:04.347Z 24 Acked 1023 Write 6 Done Done Sent false
106812023-12-10T18:54:04.347Z 25 Acked 1024 Read 5 Done Done Sent false
106822023-12-10T18:54:04.347Z 26 Acked 1025 Write 3 Done Done Sent false
106832023-12-10T18:54:04.347Z 27 Acked 1026 Read 2 Done Done Sent false
106842023-12-10T18:54:04.348Z 28 Acked 1027 Read 2 Done Done Sent false
106852023-12-10T18:54:04.348Z 29 Acked 1028 Read 3 Done Done Sent false
106862023-12-10T18:54:04.348Z 30 Acked 1029 Read 8 Sent Done Sent false
106872023-12-10T18:54:04.348Z STATES DS:0 DS:1 DS:2 TOTAL
106882023-12-10T18:54:04.348Z New 0 0 0 0
106892023-12-10T18:54:04.348Z Sent 1 0 13 14
106902023-12-10T18:54:04.348Z Done 29 30 17 76
106912023-12-10T18:54:04.348Z Skipped 0 0 0 0
106922023-12-10T18:54:04.348Z Error 0 0 0 0
106932023-12-10T18:54:04.348ZLast Flush: 0 0 0
106942023-12-10T18:54:04.348ZDownstairs last five completed:
106952023-12-10T18:54:04.349ZUpstairs last five completed: 30 29 28 27 26 25
106962023-12-10T18:54:04.349ZWrote out file "/tmp/test_repair_verify.data"
106972023-12-10T18:54:04.349Z
106982023-12-10T18:54:04.349ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
106992023-12-10T18:54:04.366ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
107002023-12-10T18:54:04.368Z 0 000-019 85 85 85 149 149 149 F F F
107012023-12-10T18:54:04.368Z 1 020-039 82 82 82 144 144 144 F F F
107022023-12-10T18:54:04.368Z 2 040-059 82 82 82 146 146 146 F F F
107032023-12-10T18:54:04.371Z 3 060-079 80 80 80 138 138 138 F F F
107042023-12-10T18:54:04.371Z 4 080-099 85 85 85 149 149 149 F F F
107052023-12-10T18:54:04.371Z 5 100-119 85 85 85 149 149 149 F F F
107062023-12-10T18:54:04.371Z 6 120-139 82 82 82 143 143 143 F F F
107072023-12-10T18:54:04.382Z 7 140-159 78 78 78 135 135 135 T T T
107082023-12-10T18:54:04.382Z 8 160-179 85 85 85 149 149 149 T T T
107092023-12-10T18:54:04.382Z 9 180-199 82 82 82 146 146 146 T T T
107102023-12-10T18:54:04.383Z 10 200-219 82 82 82 146 146 146 T T T
107112023-12-10T18:54:04.383Z 11 220-239 85 85 85 149 149 149 T T T
107122023-12-10T18:54:04.383Z 12 240-259 82 82 82 143 143 143 F F F
107132023-12-10T18:54:04.383Z 13 260-279 82 82 82 142 142 142 T T T
107142023-12-10T18:54:04.383Z 14 280-299 85 85 85 149 149 149 T T T
107152023-12-10T18:54:04.383Z 15 300-319 78 78 78 135 135 135 T T F <---
107162023-12-10T18:54:04.383Z 16 320-339 85 85 85 149 149 149 T T F <---
107172023-12-10T18:54:04.383Z 17 340-359 85 85 85 149 149 149 F F F
107182023-12-10T18:54:04.383Z 18 360-379 84 84 84 148 148 148 T T T
107192023-12-10T18:54:04.383Z 19 380-399 84 84 84 148 148 148 T T T
107202023-12-10T18:54:04.384Z 20 400-419 82 82 82 146 146 146 T T F <---
107212023-12-10T18:54:04.384Z 21 420-439 85 85 85 149 149 149 T T F <---
107222023-12-10T18:54:04.384Z 22 440-459 85 85 85 149 149 149 T T F <---
107232023-12-10T18:54:04.384Z 23 460-479 82 82 82 145 145 145 F F F
107242023-12-10T18:54:04.384Z 24 480-499 85 85 85 149 149 149 T T F <---
107252023-12-10T18:54:04.384Z 25 500-519 84 84 84 148 148 148 F F F
107262023-12-10T18:54:04.384Z 26 520-539 82 82 82 143 143 143 F F F
107272023-12-10T18:54:04.385Z 27 540-559 82 82 82 143 143 143 F F F
107282023-12-10T18:54:04.385Z 28 560-579 84 84 84 148 148 148 F F F
107292023-12-10T18:54:04.385Z 29 580-599 84 84 84 148 148 148 F F F
107302023-12-10T18:54:04.385ZMax gen: 85, Max flush: 149
107312023-12-10T18:54:04.385ZError: Difference in extent metadata found!
107322023-12-10T18:54:04.386ZA Difference in extent metadata is expected here
107332023-12-10T18:54:04.386ZOn loop 43
107342023-12-10T18:54:04.386Z
107352023-12-10T18:54:04.386ZVerifying data now
107362023-12-10T18:54:10.838ZNo change after verify
107372023-12-10T18:54:10.839ZLoop: 43 Downstairs dump after verify (and repair):
107382023-12-10T18:54:10.861ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
107392023-12-10T18:54:10.861Z 0 000-019 85 85 85 149 149 149 F F F
107402023-12-10T18:54:10.865Z 1 020-039 82 82 82 144 144 144 F F F
107412023-12-10T18:54:10.865Z 2 040-059 82 82 82 146 146 146 F F F
107422023-12-10T18:54:10.865Z 3 060-079 80 80 80 138 138 138 F F F
107432023-12-10T18:54:10.865Z 4 080-099 85 85 85 149 149 149 F F F
107442023-12-10T18:54:10.866Z 5 100-119 85 85 85 149 149 149 F F F
107452023-12-10T18:54:10.866Z 6 120-139 82 82 82 143 143 143 F F F
107462023-12-10T18:54:10.866Z 7 140-159 86 86 86 150 150 150 F F F
107472023-12-10T18:54:10.866Z 8 160-179 86 86 86 150 150 150 F F F
107482023-12-10T18:54:10.866Z 9 180-199 86 86 86 150 150 150 F F F
107492023-12-10T18:54:10.866Z 10 200-219 86 86 86 150 150 150 F F F
107502023-12-10T18:54:10.866Z 11 220-239 86 86 86 150 150 150 F F F
107512023-12-10T18:54:10.866Z 12 240-259 82 82 82 143 143 143 F F F
107522023-12-10T18:54:10.866Z 13 260-279 86 86 86 150 150 150 F F F
107532023-12-10T18:54:10.866Z 14 280-299 86 86 86 150 150 150 F F F
107542023-12-10T18:54:10.866Z 15 300-319 86 86 86 150 150 150 F F F
107552023-12-10T18:54:10.866Z 16 320-339 86 86 86 150 150 150 F F F
107562023-12-10T18:54:10.867Z 17 340-359 85 85 85 149 149 149 F F F
107572023-12-10T18:54:10.867Z 18 360-379 86 86 86 150 150 150 F F F
107582023-12-10T18:54:10.867Z 19 380-399 86 86 86 150 150 150 F F F
107592023-12-10T18:54:10.867Z 20 400-419 86 86 86 150 150 150 F F F
107602023-12-10T18:54:10.867Z 21 420-439 86 86 86 150 150 150 F F F
107612023-12-10T18:54:10.867Z 22 440-459 86 86 86 150 150 150 F F F
107622023-12-10T18:54:10.868Z 23 460-479 82 82 82 145 145 145 F F F
107632023-12-10T18:54:10.868Z 24 480-499 86 86 86 150 150 150 F F F
107642023-12-10T18:54:10.868Z 25 500-519 84 84 84 148 148 148 F F F
107652023-12-10T18:54:10.868Z 26 520-539 82 82 82 143 143 143 F F F
107662023-12-10T18:54:10.868Z 27 540-559 82 82 82 143 143 143 F F F
107672023-12-10T18:54:10.868Z 28 560-579 84 84 84 148 148 148 F F F
107682023-12-10T18:54:10.868Z 29 580-599 84 84 84 148 148 148 F F F
107692023-12-10T18:54:10.868ZMax gen: 86, Max flush: 150
107702023-12-10T18:54:10.872Z
107712023-12-10T18:54:10.872ZBegin loop 44
107722023-12-10T18:54:10.872ZDownstairs to restart: 1
107732023-12-10T18:54:10.895Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.899501005Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514}
107742023-12-10T18:54:10.895Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.899977866Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514}
107752023-12-10T18:54:10.895Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.900018274Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514}
107762023-12-10T18:54:10.901Z{"msg":"Crucible e181c2b9-a63a-4ddb-94d0-888fb10d156b has session id: 2c231f6b-1e85-458f-a218-d81125a4e12d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.900077616Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
107772023-12-10T18:54:10.901Z{"msg":"Crucible runtime is spawned
107782023-12-10T18:54:10.901ZUpstairs opts: Upstairs UUID: e181c2b9-a63a-4ddb-94d0-888fb10d156b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.900140669Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
107792023-12-10T18:54:10.901Z{"msg":"Crucible stats registered with UUID: e181c2b9-a63a-4ddb-94d0-888fb10d156b","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.900180266Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
107802023-12-10T18:54:10.901ZThe guest has requested activation with gen:88
107812023-12-10T18:54:10.901Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.900342173Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"looper":"1","session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
107822023-12-10T18:54:10.901Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.900446347Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d","task":"up_listen"}
107832023-12-10T18:54:10.902Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.900492561Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
107842023-12-10T18:54:10.902Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.900651468Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
107852023-12-10T18:54:10.902Z{"msg":"[1] e181c2b9-a63a-4ddb-94d0-888fb10d156b looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.900679266Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"looper":"1","session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
107862023-12-10T18:54:10.902Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.900701041Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"looper":"2","session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
107872023-12-10T18:54:10.902Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.900914072Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"looper":"0","session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
107882023-12-10T18:54:10.902Z{"msg":"[2] e181c2b9-a63a-4ddb-94d0-888fb10d156b looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.90099424Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"looper":"2","session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
107892023-12-10T18:54:10.902Z{"msg":"[0] e181c2b9-a63a-4ddb-94d0-888fb10d156b looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.901049036Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"looper":"0","session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
107902023-12-10T18:54:10.904Z{"msg":"e181c2b9-a63a-4ddb-94d0-888fb10d156b active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.90138881Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
107912023-12-10T18:54:10.905Z{"msg":"Set desired generation to :88","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.901443497Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
107922023-12-10T18:54:10.905Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.901513375Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
107932023-12-10T18:54:10.905Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.901809918Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
107942023-12-10T18:54:10.905Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.901874563Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
107952023-12-10T18:54:10.905Z{"msg":"[0] e181c2b9-a63a-4ddb-94d0-888fb10d156b (2c231f6b-1e85-458f-a218-d81125a4e12d) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.902342953Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
107962023-12-10T18:54:10.905Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.902513811Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
107972023-12-10T18:54:10.905Z{"msg":"[0] client is_active_req TRUE, promote! session 2c231f6b-1e85-458f-a218-d81125a4e12d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.902629264Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
107982023-12-10T18:54:10.905Z{"msg":"[2] e181c2b9-a63a-4ddb-94d0-888fb10d156b (2c231f6b-1e85-458f-a218-d81125a4e12d) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.902676863Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
107992023-12-10T18:54:10.905Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.902755395Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108002023-12-10T18:54:10.906Z{"msg":"[2] client is_active_req TRUE, promote! session 2c231f6b-1e85-458f-a218-d81125a4e12d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.902797792Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108012023-12-10T18:54:10.906Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.9032971Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108022023-12-10T18:54:10.906Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.903407797Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108032023-12-10T18:54:10.906Z{"msg":"[1] e181c2b9-a63a-4ddb-94d0-888fb10d156b (2c231f6b-1e85-458f-a218-d81125a4e12d) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.903456731Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108042023-12-10T18:54:10.906Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.903504099Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108052023-12-10T18:54:10.906Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.90355826Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108062023-12-10T18:54:10.906Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.903606084Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108072023-12-10T18:54:10.906Z{"msg":"[1] client is_active_req TRUE, promote! session 2c231f6b-1e85-458f-a218-d81125a4e12d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.903638206Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108082023-12-10T18:54:10.906Z{"msg":"e181c2b9-a63a-4ddb-94d0-888fb10d156b WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.903674799Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108092023-12-10T18:54:10.907Z{"msg":"e181c2b9-a63a-4ddb-94d0-888fb10d156b WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.903713788Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108102023-12-10T18:54:11.022Z{"msg":"[2] e181c2b9-a63a-4ddb-94d0-888fb10d156b (2c231f6b-1e85-458f-a218-d81125a4e12d) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.904129748Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108112023-12-10T18:54:11.022Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.904229421Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108122023-12-10T18:54:11.022Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:10.904317263Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108132023-12-10T18:54:11.022Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.90436001Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108142023-12-10T18:54:11.022Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.904426631Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108152023-12-10T18:54:11.023Z{"msg":"e181c2b9-a63a-4ddb-94d0-888fb10d156b WaitActive WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.904469704Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108162023-12-10T18:54:11.023Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.904540418Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108172023-12-10T18:54:11.023Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.904609024Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108182023-12-10T18:54:11.023Z{"msg":"[0] e181c2b9-a63a-4ddb-94d0-888fb10d156b (2c231f6b-1e85-458f-a218-d81125a4e12d) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.904638565Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108192023-12-10T18:54:11.023Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.904680239Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108202023-12-10T18:54:11.023Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:10.904702512Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108212023-12-10T18:54:11.023Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.904742051Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108222023-12-10T18:54:11.023Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.904764166Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108232023-12-10T18:54:11.024Z{"msg":"e181c2b9-a63a-4ddb-94d0-888fb10d156b WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.90480529Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108242023-12-10T18:54:11.024Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.904828125Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108252023-12-10T18:54:11.024Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.904849295Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108262023-12-10T18:54:11.024Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.904869055Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108272023-12-10T18:54:11.024Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.904893825Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108282023-12-10T18:54:11.024Z{"msg":"e181c2b9-a63a-4ddb-94d0-888fb10d156b WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.904914937Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108292023-12-10T18:54:11.024Z{"msg":"[1] e181c2b9-a63a-4ddb-94d0-888fb10d156b (2c231f6b-1e85-458f-a218-d81125a4e12d) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.905263059Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108302023-12-10T18:54:11.024Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.905328523Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108312023-12-10T18:54:11.025Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:10.905351869Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108322023-12-10T18:54:11.025Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.905371946Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108332023-12-10T18:54:11.025Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.905392424Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108342023-12-10T18:54:11.025Z{"msg":"e181c2b9-a63a-4ddb-94d0-888fb10d156b WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.90541329Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108352023-12-10T18:54:11.025Z{"msg":"[0]R flush_numbers[0..12]: [149, 144, 146, 138, 149, 149, 143, 150, 150, 150, 150, 150]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.90550974Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108362023-12-10T18:54:11.025Z{"msg":"[0]R generation[0..12]: [85, 82, 82, 80, 85, 85, 82, 86, 86, 86, 86, 86]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.905558818Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108372023-12-10T18:54:11.025Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.905583915Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108382023-12-10T18:54:11.025Z{"msg":"[1]R flush_numbers[0..12]: [149, 144, 146, 138, 149, 149, 143, 150, 150, 150, 150, 150]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.90560553Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108392023-12-10T18:54:11.027Z{"msg":"[1]R generation[0..12]: [85, 82, 82, 80, 85, 85, 82, 86, 86, 86, 86, 86]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.905646816Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108402023-12-10T18:54:11.029Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.905670501Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108412023-12-10T18:54:11.029Z{"msg":"[2]R flush_numbers[0..12]: [149, 144, 146, 138, 149, 149, 143, 150, 150, 150, 150, 150]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.905691665Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108422023-12-10T18:54:11.029Z{"msg":"[2]R generation[0..12]: [85, 82, 82, 80, 85, 85, 82, 86, 86, 86, 86, 86]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.905714327Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108432023-12-10T18:54:11.029Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:88
108442023-12-10T18:54:11.029Z,"time":"2023-12-10T18:54:10.905736184Z","hostname":"Wait for a query_work_queue command to finish before sending IO
108452023-12-10T18:54:11.030Zip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108462023-12-10T18:54:11.030Z{"msg":"Max found gen is 87","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.905761667Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108472023-12-10T18:54:11.030Z{"msg":"Generation requested: 88 >= found:87","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.905781796Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108482023-12-10T18:54:11.030Z{"msg":"Next flush: 151","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.905828129Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108492023-12-10T18:54:11.030Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.905849846Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108502023-12-10T18:54:11.031Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.905870712Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108512023-12-10T18:54:11.031Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.905941903Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108522023-12-10T18:54:11.031Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.905982313Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108532023-12-10T18:54:11.031Z{"msg":"e181c2b9-a63a-4ddb-94d0-888fb10d156b is now active with session: 2c231f6b-1e85-458f-a218-d81125a4e12d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.906021265Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108542023-12-10T18:54:11.031Z{"msg":"e181c2b9-a63a-4ddb-94d0-888fb10d156b Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.906081506Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108552023-12-10T18:54:11.031Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.906106145Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108562023-12-10T18:54:11.031Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.906125753Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108572023-12-10T18:54:11.031Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.906153711Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108582023-12-10T18:54:11.032Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.906188836Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108592023-12-10T18:54:11.032Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.906248817Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108602023-12-10T18:54:11.032Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.906479296Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108612023-12-10T18:54:11.033Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.906534256Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108622023-12-10T18:54:11.033Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.90676559Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108632023-12-10T18:54:11.033Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.906827998Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108642023-12-10T18:54:11.033Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.906870971Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108652023-12-10T18:54:11.033Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:10.906893649Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1514,"session_id":"2c231f6b-1e85-458f-a218-d81125a4e12d"}
108662023-12-10T18:54:11.033ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
108672023-12-10T18:54:11.033ZLoading write count information from file "/tmp/test_repair_verify.data"
108682023-12-10T18:54:11.033ZRun Repair workload
108692023-12-10T18:54:11.034Z01/30 Write block 93 len 20480 data: 7 7 6 5 5
108702023-12-10T18:54:11.034Z02/30 Write block 260 len 16384 data: 3 4 6 8
108712023-12-10T18:54:11.034Z03/30 Flush
108722023-12-10T18:54:11.034Z04/30 Write block 424 len 32768 data: 8 6 5 6 5 4 4 3
108732023-12-10T18:54:11.034Z05/30 Read block 90 len 32768
108742023-12-10T18:54:11.041Z06/30 Write block 65 len 36864 data: 6 7 5 7 6 5 6 5 6
108752023-12-10T18:54:11.041Z07/30 Read block 257 len 32768
108762023-12-10T18:54:11.042Z08/30 Write block 34 len 24576 data: 7 8 9 8 7 8
108772023-12-10T18:54:11.042Z09/30 Read block 387 len 8192
108782023-12-10T18:54:11.044Z10/30 Read block 215 len 20480
108792023-12-10T18:54:11.044Z11/30 Write block 159 len 24576 data: 7 7 7 10 11 11
108802023-12-10T18:54:11.044Z12/30 Read block 199 len 40960
108812023-12-10T18:54:11.044Z13/30 Write block 244 len 28672 data: 12 11 12 10 10 11 9
108822023-12-10T18:54:11.044Z14/30 Flush
108832023-12-10T18:54:11.044Z15/30 Flush
108842023-12-10T18:54:11.044Z16/30 Write block 570 len 40960 data: 8 6 4 5 7 8 8 8 7 8
108852023-12-10T18:54:11.044Z17/30 Read block 294 len 36864
108862023-12-10T18:54:11.045Z18/30 Flush
108872023-12-10T18:54:11.045Z19/30 Write block 405 len 20480 data: 9 8 8 8 6
108882023-12-10T18:54:11.045Z20/30 Write block 153 len 12288 data: 5 8 4
108892023-12-10T18:54:11.045Z21/30 Read block 103 len 20480
108902023-12-10T18:54:11.045Z22/30 Read block 230 len 24576
108912023-12-10T18:54:11.045Z23/30 Read block 302 len 32768
108922023-12-10T18:54:11.045Z24/30 Flush
108932023-12-10T18:54:11.045Z25/30 Write block 196 len 16384 data: 12 14 13 12
108942023-12-10T18:54:11.045Z26/30 Write block 592 len 16384 data: 5 4 4 3
108952023-12-10T18:54:11.045Z27/30 Write block 24 len 12288 data: 6 6 8
108962023-12-10T18:54:11.045Z28/30 Read block 568 len 40960
108972023-12-10T18:54:11.045Z29/30 Write block 415 len 40960 data: 4 5 5 7 7 6 7 7 7 9
108982023-12-10T18:54:11.045Z30/30 Read block 118 len 12288
108992023-12-10T18:54:11.045Z----------------------------------------------------------------
109002023-12-10T18:54:11.045Z Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30
109012023-12-10T18:54:11.045ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
109022023-12-10T18:54:11.045Z 1 Acked 1000 Write 5 Done Sent Done false
109032023-12-10T18:54:11.046Z 2 Acked 1001 Write 4 Done Sent Done false
109042023-12-10T18:54:11.046Z 3 Acked 1002 Flush 0 Done Sent Done false
109052023-12-10T18:54:11.046Z 4 Acked 1003 Write 8 Done Sent Done false
109062023-12-10T18:54:11.046Z 5 Acked 1004 Read 8 Done Sent Done false
109072023-12-10T18:54:11.046Z 6 Acked 1005 Write 9 Done Sent Done false
109082023-12-10T18:54:11.046Z 7 Acked 1006 Read 8 Done Sent Done false
109092023-12-10T18:54:11.046Z 8 Acked 1007 Write 6 Done Sent Done false
109102023-12-10T18:54:11.046Z 9 Acked 1008 Read 2 Done Sent Done false
109112023-12-10T18:54:11.046Z 10 Acked 1009 Read 5 Done Sent Done false
109122023-12-10T18:54:11.046Z 11 Acked 1010 Write 6 Done Sent Done false
109132023-12-10T18:54:11.046Z 12 Acked 1011 Read 10 Done Sent Done false
109142023-12-10T18:54:11.046Z 13 Acked 1012 Write 7 Done Sent Done false
109152023-12-10T18:54:11.046Z 14 Acked 1013 Flush 0 Done Sent Done false
109162023-12-10T18:54:11.046Z 15 Acked 1014 Flush 0 Done Sent Done false
109172023-12-10T18:54:11.046Z 16 Acked 1015 Write 10 Done Sent Done false
109182023-12-10T18:54:11.046Z 17 Acked 1016 Read 9 Done Sent Done false
109192023-12-10T18:54:11.046Z 18 Acked 1017 Flush 0 Done Sent Done false
109202023-12-10T18:54:11.046Z 19 Acked 1018 Write 5 Done Sent Done false
109212023-12-10T18:54:11.047Z 20 Acked 1019 Write 3 Done Sent Done false
109222023-12-10T18:54:11.047Z 21 Acked 1020 Read 5 Done Sent Done false
109232023-12-10T18:54:11.047Z 22 Acked 1021 Read 6 Done Sent Done false
109242023-12-10T18:54:11.047Z 23 Acked 1022 Read 8 Done Sent Done false
109252023-12-10T18:54:11.047Z 24 Acked 1023 Flush 0 Done Sent Done false
109262023-12-10T18:54:11.047Z 25 Acked 1024 Write 4 Done Sent Done false
109272023-12-10T18:54:11.047Z 26 Acked 1025 Write 4 Done Sent Done false
109282023-12-10T18:54:11.048Z 27 Acked 1026 Write 3 Done Sent Done false
109292023-12-10T18:54:11.048Z 28 Acked 1027 Read 10 Done Sent Done false
109302023-12-10T18:54:11.048Z 29 Acked 1028 Write 10 Done Sent Done false
109312023-12-10T18:54:11.048Z 30 Acked 1029 Read 3 Done Sent Sent false
109322023-12-10T18:54:11.048Z STATES DS:0 DS:1 DS:2 TOTAL
109332023-12-10T18:54:11.048Z New 0 0 0 0
109342023-12-10T18:54:11.048Z Sent 0 30 1 31
109352023-12-10T18:54:11.048Z Done 30 0 29 59
109362023-12-10T18:54:11.048Z Skipped 0 0 0 0
109372023-12-10T18:54:11.048Z Error 0 0 0 0
109382023-12-10T18:54:11.048ZLast Flush: 1023 0 1023
109392023-12-10T18:54:11.049ZDownstairs last five completed:
109402023-12-10T18:54:11.049ZUpstairs last five completed: 30 29 28 27 26 25
109412023-12-10T18:54:11.049ZWrote out file "/tmp/test_repair_verify.data"
109422023-12-10T18:54:11.049Z
109432023-12-10T18:54:11.049ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
109442023-12-10T18:54:11.049ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
109452023-12-10T18:54:11.049Z 0 000-019 85 85 85 149 149 149 F F F
109462023-12-10T18:54:11.050Z 1 020-039 88 82 88 152 144 152 T F T <---
109472023-12-10T18:54:11.050Z 2 040-059 82 82 82 146 146 146 F F F
109482023-12-10T18:54:11.050Z 3 060-079 88 80 88 152 138 152 F F F <---
109492023-12-10T18:54:11.050Z 4 080-099 88 85 88 151 149 151 F F F <---
109502023-12-10T18:54:11.050Z 5 100-119 85 85 85 149 149 149 F F F
109512023-12-10T18:54:11.051Z 6 120-139 82 82 82 143 143 143 F F F
109522023-12-10T18:54:11.051Z 7 140-159 88 86 88 155 150 155 F F F <---
109532023-12-10T18:54:11.051Z 8 160-179 88 86 88 152 150 152 F F F <---
109542023-12-10T18:54:11.051Z 9 180-199 86 86 86 150 150 150 T F T <---
109552023-12-10T18:54:11.051Z 10 200-219 86 86 86 150 150 150 F F F
109562023-12-10T18:54:11.051Z 11 220-239 86 86 86 150 150 150 F F F
109572023-12-10T18:54:11.051Z 12 240-259 88 82 88 152 143 152 F F F <---
109582023-12-10T18:54:11.051Z 13 260-279 88 86 88 151 150 151 F F F <---
109592023-12-10T18:54:11.051Z 14 280-299 86 86 86 150 150 150 F F F
109602023-12-10T18:54:11.051Z 15 300-319 86 86 86 150 150 150 F F F
109612023-12-10T18:54:11.051Z 16 320-339 86 86 86 150 150 150 F F F
109622023-12-10T18:54:11.051Z 17 340-359 85 85 85 149 149 149 F F F
109632023-12-10T18:54:11.051Z 18 360-379 86 86 86 150 150 150 F F F
109642023-12-10T18:54:11.051Z 19 380-399 86 86 86 150 150 150 F F F
109652023-12-10T18:54:11.051Z 20 400-419 88 86 88 155 150 155 T F T <---
109662023-12-10T18:54:11.051Z 21 420-439 88 86 88 152 150 152 T F T <---
109672023-12-10T18:54:11.052Z 22 440-459 86 86 86 150 150 150 F F F
109682023-12-10T18:54:11.052Z 23 460-479 82 82 82 145 145 145 F F F
109692023-12-10T18:54:11.052Z 24 480-499 86 86 86 150 150 150 F F F
109702023-12-10T18:54:11.052Z 25 500-519 84 84 84 148 148 148 F F F
109712023-12-10T18:54:11.052Z 26 520-539 82 82 82 143 143 143 F F F
109722023-12-10T18:54:11.052Z 27 540-559 82 82 82 143 143 143 F F F
109732023-12-10T18:54:11.052Z 28 560-579 88 84 88 154 148 154 F F F <---
109742023-12-10T18:54:11.057Z 29 580-599 84 84 84 148 148 148 T F T <---
109752023-12-10T18:54:11.057ZMax gen: 88, Max flush: 155
109762023-12-10T18:54:11.057ZError: Difference in extent metadata found!
109772023-12-10T18:54:11.057ZA Difference in extent metadata is expected here
109782023-12-10T18:54:11.057ZOn loop 44
109792023-12-10T18:54:11.057Z
109802023-12-10T18:54:11.057ZVerifying data now
109812023-12-10T18:54:16.215ZNo change after verify
109822023-12-10T18:54:16.215ZLoop: 44 Downstairs dump after verify (and repair):
109832023-12-10T18:54:16.238ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
109842023-12-10T18:54:16.238Z 0 000-019 85 85 85 149 149 149 F F F
109852023-12-10T18:54:16.241Z 1 020-039 89 89 89 156 156 156 F F F
109862023-12-10T18:54:16.241Z 2 040-059 82 82 82 146 146 146 F F F
109872023-12-10T18:54:16.241Z 3 060-079 88 88 88 152 152 152 F F F
109882023-12-10T18:54:16.241Z 4 080-099 88 88 88 151 151 151 F F F
109892023-12-10T18:54:16.241Z 5 100-119 85 85 85 149 149 149 F F F
109902023-12-10T18:54:16.241Z 6 120-139 82 82 82 143 143 143 F F F
109912023-12-10T18:54:16.241Z 7 140-159 88 88 88 155 155 155 F F F
109922023-12-10T18:54:16.241Z 8 160-179 88 88 88 152 152 152 F F F
109932023-12-10T18:54:16.241Z 9 180-199 89 89 89 156 156 156 F F F
109942023-12-10T18:54:16.241Z 10 200-219 86 86 86 150 150 150 F F F
109952023-12-10T18:54:16.241Z 11 220-239 86 86 86 150 150 150 F F F
109962023-12-10T18:54:16.241Z 12 240-259 88 88 88 152 152 152 F F F
109972023-12-10T18:54:16.244Z 13 260-279 88 88 88 151 151 151 F F F
109982023-12-10T18:54:16.244Z 14 280-299 86 86 86 150 150 150 F F F
109992023-12-10T18:54:16.244Z 15 300-319 86 86 86 150 150 150 F F F
110002023-12-10T18:54:16.244Z 16 320-339 86 86 86 150 150 150 F F F
110012023-12-10T18:54:16.244Z 17 340-359 85 85 85 149 149 149 F F F
110022023-12-10T18:54:16.244Z 18 360-379 86 86 86 150 150 150 F F F
110032023-12-10T18:54:16.244Z 19 380-399 86 86 86 150 150 150 F F F
110042023-12-10T18:54:16.244Z 20 400-419 89 89 89 156 156 156 F F F
110052023-12-10T18:54:16.244Z 21 420-439 89 89 89 156 156 156 F F F
110062023-12-10T18:54:16.244Z 22 440-459 86 86 86 150 150 150 F F F
110072023-12-10T18:54:16.244Z 23 460-479 82 82 82 145 145 145 F F F
110082023-12-10T18:54:16.244Z 24 480-499 86 86 86 150 150 150 F F F
110092023-12-10T18:54:16.244Z 25 500-519 84 84 84 148 148 148 F F F
110102023-12-10T18:54:16.244Z 26 520-539 82 82 82 143 143 143 F F F
110112023-12-10T18:54:16.244Z 27 540-559 82 82 82 143 143 143 F F F
110122023-12-10T18:54:16.245Z 28 560-579 88 88 88 154 154 154 F F F
110132023-12-10T18:54:16.245Z 29 580-599 89 89 89 156 156 156 F F F
110142023-12-10T18:54:16.245ZMax gen: 89, Max flush: 156
110152023-12-10T18:54:16.248Z
110162023-12-10T18:54:16.248ZBegin loop 45
110172023-12-10T18:54:16.248ZDownstairs to restart: 1
110182023-12-10T18:54:16.271Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.276233659Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522}
110192023-12-10T18:54:16.271Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.27664321Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522}
110202023-12-10T18:54:16.276Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.2766713Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522}
110212023-12-10T18:54:16.276Z{"msg":"Crucible 4206eada-bd16-4429-9c5e-5d7ee8c0afcf has session id: 3339ecae-a3d4-41c6-86b5-963a4129bcbd","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.27672094Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110222023-12-10T18:54:16.276Z{"msg":"Upstairs opts: Upstairs UUID: 4206eada-bd16-4429-9c5e-5d7ee8c0afcf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.276767177Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110232023-12-10T18:54:16.276Z{"msg":"Crucible stats registered with UUID: 4206eada-bd16-4429-9c5e-5d7ee8c0afcf","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.276811711Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110242023-12-10T18:54:16.276ZCrucible runtime is spawned
110252023-12-10T18:54:16.276ZThe guest has requested activation with gen:90
110262023-12-10T18:54:16.276Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.277185555Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd","task":"up_listen"}
110272023-12-10T18:54:16.276Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.277267378Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"looper":"2","session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110282023-12-10T18:54:16.276Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.277412199Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110292023-12-10T18:54:16.276Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.27751203Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110302023-12-10T18:54:16.276Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.277620737Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"looper":"0","session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110312023-12-10T18:54:16.276Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.277686543Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"looper":"1","session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110322023-12-10T18:54:16.276Z{"msg":"[2] 4206eada-bd16-4429-9c5e-5d7ee8c0afcf looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.277731056Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"looper":"2","session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110332023-12-10T18:54:16.276Z{"msg":"[1] 4206eada-bd16-4429-9c5e-5d7ee8c0afcf looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.277780342Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"looper":"1","session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110342023-12-10T18:54:16.276Z{"msg":"[0] 4206eada-bd16-4429-9c5e-5d7ee8c0afcf looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.277834344Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"looper":"0","session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110352023-12-10T18:54:16.277Z{"msg":"4206eada-bd16-4429-9c5e-5d7ee8c0afcf active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.278111205Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110362023-12-10T18:54:16.277Z{"msg":"Set desired generation to :90","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.278180421Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110372023-12-10T18:54:16.277Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.278317505Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110382023-12-10T18:54:16.277Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.278560266Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110392023-12-10T18:54:16.277Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.278709619Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110402023-12-10T18:54:16.277Z{"msg":"[2] 4206eada-bd16-4429-9c5e-5d7ee8c0afcf (3339ecae-a3d4-41c6-86b5-963a4129bcbd) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.279126213Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110412023-12-10T18:54:16.277Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.279197189Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110422023-12-10T18:54:16.277Z{"msg":"[2] client is_active_req TRUE, promote! session 3339ecae-a3d4-41c6-86b5-963a4129bcbd","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.279239154Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110432023-12-10T18:54:16.277Z{"msg":"[0] 4206eada-bd16-4429-9c5e-5d7ee8c0afcf (3339ecae-a3d4-41c6-86b5-963a4129bcbd) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.279283306Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110442023-12-10T18:54:16.277Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.279320399Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110452023-12-10T18:54:16.277Z{"msg":"[0] client is_active_req TRUE, promote! session 3339ecae-a3d4-41c6-86b5-963a4129bcbd","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.279360434Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110462023-12-10T18:54:16.277Z{"msg":"[1] 4206eada-bd16-4429-9c5e-5d7ee8c0afcf (3339ecae-a3d4-41c6-86b5-963a4129bcbd) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.27968533Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110472023-12-10T18:54:16.277Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.279776479Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110482023-12-10T18:54:16.277Z{"msg":"[1] client is_active_req TRUE, promote! session 3339ecae-a3d4-41c6-86b5-963a4129bcbd","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.279853431Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110492023-12-10T18:54:16.277Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.279942613Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110502023-12-10T18:54:16.277Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.280131806Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110512023-12-10T18:54:16.285Z{"msg":"4206eada-bd16-4429-9c5e-5d7ee8c0afcf WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.280235868Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110522023-12-10T18:54:16.285Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.280282201Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110532023-12-10T18:54:16.285Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.280348995Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110542023-12-10T18:54:16.285Z{"msg":"4206eada-bd16-4429-9c5e-5d7ee8c0afcf WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.280390326Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110552023-12-10T18:54:16.285Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.28044609Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110562023-12-10T18:54:16.285Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.280492014Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110572023-12-10T18:54:16.285Z{"msg":"4206eada-bd16-4429-9c5e-5d7ee8c0afcf WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.280658187Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110582023-12-10T18:54:16.285Z{"msg":"[2] 4206eada-bd16-4429-9c5e-5d7ee8c0afcf (3339ecae-a3d4-41c6-86b5-963a4129bcbd) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.28087944Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110592023-12-10T18:54:16.286Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.280977677Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110602023-12-10T18:54:16.286Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:16.281003311Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110612023-12-10T18:54:16.286Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.281027737Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110622023-12-10T18:54:16.286Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.281085522Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110632023-12-10T18:54:16.286Z{"msg":"4206eada-bd16-4429-9c5e-5d7ee8c0afcf WaitActive WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.281112515Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110642023-12-10T18:54:16.286Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.281189806Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110652023-12-10T18:54:16.286Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.281228349Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110662023-12-10T18:54:16.286Z{"msg":"[1] 4206eada-bd16-4429-9c5e-5d7ee8c0afcf (3339ecae-a3d4-41c6-86b5-963a4129bcbd) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.28130376Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110672023-12-10T18:54:16.286Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.281329896Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110682023-12-10T18:54:16.286Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:16.281359494Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110692023-12-10T18:54:16.286Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.281414817Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110702023-12-10T18:54:16.286Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.28143948Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110712023-12-10T18:54:16.286Z{"msg":"4206eada-bd16-4429-9c5e-5d7ee8c0afcf WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.28146205Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110722023-12-10T18:54:16.286Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.281518012Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110732023-12-10T18:54:16.286Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.28155904Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110742023-12-10T18:54:16.287Z{"msg":"[0] 4206eada-bd16-4429-9c5e-5d7ee8c0afcf (3339ecae-a3d4-41c6-86b5-963a4129bcbd) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.281621Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110752023-12-10T18:54:16.287Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.281647301Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110762023-12-10T18:54:16.287Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:16.281701701Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110772023-12-10T18:54:16.287Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.281726471Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110782023-12-10T18:54:16.287Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.281782211Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110792023-12-10T18:54:16.287Z{"msg":"4206eada-bd16-4429-9c5e-5d7ee8c0afcf WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.281808128Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110802023-12-10T18:54:16.287Z{"msg":"[0]R flush_numbers[0..12]: [149, 156, 146, 152, 151, 149, 143, 155, 152, 156, 150, 150]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.281835149Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"The guest has finished waiting for activation with:90
110812023-12-10T18:54:16.289ZWait for a query_work_queue command to finish before sending IO
110822023-12-10T18:54:16.289Z3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110832023-12-10T18:54:16.289Z{"msg":"[0]R generation[0..12]: [85, 89, 82, 88, 88, 85, 82, 88, 88, 89, 86, 86]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.281928072Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110842023-12-10T18:54:16.289Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.282000646Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110852023-12-10T18:54:16.289Z{"msg":"[1]R flush_numbers[0..12]: [149, 156, 146, 152, 151, 149, 143, 155, 152, 156, 150, 150]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.282027261Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110862023-12-10T18:54:16.289Z{"msg":"[1]R generation[0..12]: [85, 89, 82, 88, 88, 85, 82, 88, 88, 89, 86, 86]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.282099079Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110872023-12-10T18:54:16.289Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.28213072Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110882023-12-10T18:54:16.289Z{"msg":"[2]R flush_numbers[0..12]: [149, 156, 146, 152, 151, 149, 143, 155, 152, 156, 150, 150]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.282155915Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110892023-12-10T18:54:16.289Z{"msg":"[2]R generation[0..12]: [85, 89, 82, 88, 88, 85, 82, 88, 88, 89, 86, 86]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.282177332Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110902023-12-10T18:54:16.289Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.282236072Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110912023-12-10T18:54:16.289Z{"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.282265566Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110922023-12-10T18:54:16.290Z{"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.282319606Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110932023-12-10T18:54:16.290Z{"msg":"Next flush: 157","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.282343933Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110942023-12-10T18:54:16.290Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.282414444Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110952023-12-10T18:54:16.290Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.282440577Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110962023-12-10T18:54:16.290Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.28250574Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110972023-12-10T18:54:16.290Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.282589689Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110982023-12-10T18:54:16.290Z{"msg":"4206eada-bd16-4429-9c5e-5d7ee8c0afcf is now active with session: 3339ecae-a3d4-41c6-86b5-963a4129bcbd","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.282662182Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
110992023-12-10T18:54:16.290Z{"msg":"4206eada-bd16-4429-9c5e-5d7ee8c0afcf Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.282691894Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
111002023-12-10T18:54:16.290Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.282734973Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
111012023-12-10T18:54:16.290Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.282768117Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
111022023-12-10T18:54:16.290Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.282832652Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
111032023-12-10T18:54:16.290Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.282860806Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
111042023-12-10T18:54:16.290Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.282881225Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
111052023-12-10T18:54:16.290Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
111062023-12-10T18:54:16.290Z,"time":"2023-12-10T18:54:16.282938433Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
111072023-12-10T18:54:16.290Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.283080873Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
111082023-12-10T18:54:16.291Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.283153175Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
111092023-12-10T18:54:16.291Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.283222337Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
111102023-12-10T18:54:16.291Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.283288878Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
111112023-12-10T18:54:16.291Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:16.283353339Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1522,"session_id":"3339ecae-a3d4-41c6-86b5-963a4129bcbd"}
111122023-12-10T18:54:16.291ZLoading write count information from file "/tmp/test_repair_verify.data"
111132023-12-10T18:54:16.291ZRun Repair workload
111142023-12-10T18:54:16.291Z01/30 Write block 576 len 20480 data: 9 9 8 9 8
111152023-12-10T18:54:16.291Z02/30 Flush
111162023-12-10T18:54:16.295Z03/30 Write block 391 len 8192 data: 5 6
111172023-12-10T18:54:16.295Z04/30 Read block 256 len 40960
111182023-12-10T18:54:16.295Z05/30 Read block 64 len 36864
111192023-12-10T18:54:16.296Z06/30 Read block 326 len 16384
111202023-12-10T18:54:16.296Z07/30 Read block 90 len 40960
111212023-12-10T18:54:16.299Z08/30 Flush
111222023-12-10T18:54:16.299Z09/30 Flush
111232023-12-10T18:54:16.302Z10/30 Write block 234 len 32768 data: 11 9 9 10 11 14 13 14
111242023-12-10T18:54:16.302Z11/30 Flush
111252023-12-10T18:54:16.305Z12/30 Write block 216 len 4096 data: 11
111262023-12-10T18:54:16.305Z13/30 Read block 374 len 12288
111272023-12-10T18:54:16.308Z14/30 Read block 466 len 4096
111282023-12-10T18:54:16.308Z15/30 Write block 408 len 40960 data: 9 7 7 5 3 2 3 5 6 6
111292023-12-10T18:54:16.308Z16/30 Read block 526 len 8192
111302023-12-10T18:54:16.311Z17/30 Flush
111312023-12-10T18:54:16.314Z18/30 Write block 27 len 32768 data: 6 6 5 6 7 10 7 8
111322023-12-10T18:54:16.314Z19/30 Write block 14 len 28672 data: 13 12 10 11 9 10 8
111332023-12-10T18:54:16.317Z20/30 Write block 330 len 40960 data: 6 6 8 8 7 7 7 8 7 8
111342023-12-10T18:54:16.317Z21/30 Flush
111352023-12-10T18:54:16.320Z22/30 Write block 130 len 4096 data: 8
111362023-12-10T18:54:16.320Z23/30 Read block 104 len 32768
111372023-12-10T18:54:16.323Z24/30 Write block 369 len 40960 data: 5 4 5 6 7 8 11 11 10 14
111382023-12-10T18:54:16.326Z25/30 Write block 548 len 12288 data: 9 6 5
111392023-12-10T18:54:16.326Z26/30 Write block 306 len 40960 data: 7 5 5 6 3 3 3 3 6 5
111402023-12-10T18:54:16.329Z27/30 Write block 327 len 28672 data: 9 8 7 7 7 9 9
111412023-12-10T18:54:16.329Z28/30 Read block 181 len 16384
111422023-12-10T18:54:16.332Z29/30 Write block 67 len 24576 data: 6 8 7 6 7 6
111432023-12-10T18:54:16.332Z30/30 Read block 68 len 20480
111442023-12-10T18:54:16.336Z----------------------------------------------------------------
111452023-12-10T18:54:16.336Z Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:28
111462023-12-10T18:54:16.336ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
111472023-12-10T18:54:16.336Z 3 Acked 1002 Write 2 Done Sent Done false
111482023-12-10T18:54:16.336Z 4 Acked 1003 Read 10 Done Sent Done false
111492023-12-10T18:54:16.336Z 5 Acked 1004 Read 9 Done Sent Done false
111502023-12-10T18:54:16.336Z 6 Acked 1005 Read 4 Done Sent Done false
111512023-12-10T18:54:16.336Z 7 Acked 1006 Read 10 Done Sent Done false
111522023-12-10T18:54:16.336Z 8 Acked 1007 Flush 0 Done Sent Done false
111532023-12-10T18:54:16.336Z 9 Acked 1008 Flush 0 Done Sent Done false
111542023-12-10T18:54:16.336Z 10 Acked 1009 Write 8 Done Sent Done false
111552023-12-10T18:54:16.336Z 11 Acked 1010 Flush 0 Done Sent Done false
111562023-12-10T18:54:16.336Z 12 Acked 1011 Write 1 Done Sent Done false
111572023-12-10T18:54:16.336Z 13 Acked 1012 Read 3 Done Sent Done false
111582023-12-10T18:54:16.337Z 14 Acked 1013 Read 1 Done Sent Done false
111592023-12-10T18:54:16.337Z 15 Acked 1014 Write 10 Done Sent Done false
111602023-12-10T18:54:16.337Z 16 Acked 1015 Read 2 Done Sent Done false
111612023-12-10T18:54:16.337Z 17 Acked 1016 Flush 0 Done Sent Done false
111622023-12-10T18:54:16.337Z 18 Acked 1017 Write 8 Done Sent Done false
111632023-12-10T18:54:16.337Z 19 Acked 1018 Write 7 Done Sent Done false
111642023-12-10T18:54:16.337Z 20 Acked 1019 Write 10 Done Sent Done false
111652023-12-10T18:54:16.337Z 21 Acked 1020 Flush 0 Done Sent Done false
111662023-12-10T18:54:16.337Z 22 Acked 1021 Write 1 Done Sent Done false
111672023-12-10T18:54:16.337Z 23 Acked 1022 Read 8 Done Sent Done false
111682023-12-10T18:54:16.337Z 24 Acked 1023 Write 10 Done Sent Done false
111692023-12-10T18:54:16.337Z 25 Acked 1024 Write 3 Done Sent Done false
111702023-12-10T18:54:16.337Z 26 Acked 1025 Write 10 Done Sent Done false
111712023-12-10T18:54:16.337Z 27 Acked 1026 Write 7 Done Sent Done false
111722023-12-10T18:54:16.337Z 28 Acked 1027 Read 4 Done Sent Done false
111732023-12-10T18:54:16.337Z 29 Acked 1028 Write 6 Done Sent Done false
111742023-12-10T18:54:16.337Z 30 Acked 1029 Read 5 Done Sent Done false
111752023-12-10T18:54:16.338Z STATES DS:0 DS:1 DS:2 TOTAL
111762023-12-10T18:54:16.338Z New 0 0 0 0
111772023-12-10T18:54:16.338Z Sent 0 28 0 28
111782023-12-10T18:54:16.338Z Done 28 0 28 56
111792023-12-10T18:54:16.338Z Skipped 0 0 0 0
111802023-12-10T18:54:16.338Z Error 0 0 0 0
111812023-12-10T18:54:16.338ZLast Flush: 1020 1001 1020
111822023-12-10T18:54:16.338ZDownstairs last five completed: 1001 1000
111832023-12-10T18:54:16.338ZUpstairs last five completed: 30 29 28 27 26 25
111842023-12-10T18:54:16.338ZWrote out file "/tmp/test_repair_verify.data"
111852023-12-10T18:54:16.341Z
111862023-12-10T18:54:16.345ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
111872023-12-10T18:54:16.371ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
111882023-12-10T18:54:16.371Z 0 000-019 90 85 90 162 149 162 F F F <---
111892023-12-10T18:54:16.377Z 1 020-039 90 89 90 162 156 162 F F F <---
111902023-12-10T18:54:16.377Z 2 040-059 82 82 82 146 146 146 F F F
111912023-12-10T18:54:16.377Z 3 060-079 88 88 88 152 152 152 T F T <---
111922023-12-10T18:54:16.377Z 4 080-099 88 88 88 151 151 151 F F F
111932023-12-10T18:54:16.377Z 5 100-119 85 85 85 149 149 149 F F F
111942023-12-10T18:54:16.377Z 6 120-139 82 82 82 143 143 143 T F T <---
111952023-12-10T18:54:16.377Z 7 140-159 88 88 88 155 155 155 F F F
111962023-12-10T18:54:16.377Z 8 160-179 88 88 88 152 152 152 F F F
111972023-12-10T18:54:16.377Z 9 180-199 89 89 89 156 156 156 F F F
111982023-12-10T18:54:16.377Z 10 200-219 90 86 90 161 150 161 F F F <---
111992023-12-10T18:54:16.377Z 11 220-239 90 86 90 160 150 160 F F F <---
112002023-12-10T18:54:16.377Z 12 240-259 90 88 90 160 152 160 F F F <---
112012023-12-10T18:54:16.378Z 13 260-279 88 88 88 151 151 151 F F F
112022023-12-10T18:54:16.378Z 14 280-299 86 86 86 150 150 150 F F F
112032023-12-10T18:54:16.378Z 15 300-319 86 86 86 150 150 150 T F T <---
112042023-12-10T18:54:16.378Z 16 320-339 90 86 90 162 150 162 T F T <---
112052023-12-10T18:54:16.378Z 17 340-359 85 85 85 149 149 149 F F F
112062023-12-10T18:54:16.378Z 18 360-379 86 86 86 150 150 150 T F T <---
112072023-12-10T18:54:16.378Z 19 380-399 90 86 90 158 150 158 F F F <---
112082023-12-10T18:54:16.378Z 20 400-419 90 89 90 161 156 161 F F F <---
112092023-12-10T18:54:16.378Z 21 420-439 89 89 89 156 156 156 F F F
112102023-12-10T18:54:16.378Z 22 440-459 86 86 86 150 150 150 F F F
112112023-12-10T18:54:16.378Z 23 460-479 82 82 82 145 145 145 F F F
112122023-12-10T18:54:16.378Z 24 480-499 86 86 86 150 150 150 F F F
112132023-12-10T18:54:16.378Z 25 500-519 84 84 84 148 148 148 F F F
112142023-12-10T18:54:16.378Z 26 520-539 82 82 82 143 143 143 F F F
112152023-12-10T18:54:16.378Z 27 540-559 82 82 82 143 143 143 T F T <---
112162023-12-10T18:54:16.378Z 28 560-579 90 90 90 157 157 157 F F F
112172023-12-10T18:54:16.378Z 29 580-599 90 90 90 157 157 157 F F F
112182023-12-10T18:54:16.379ZMax gen: 90, Max flush: 162
112192023-12-10T18:54:16.379ZError: Difference in extent metadata found!
112202023-12-10T18:54:16.382ZA Difference in extent metadata is expected here
112212023-12-10T18:54:16.382ZOn loop 45
112222023-12-10T18:54:16.382Z
112232023-12-10T18:54:16.382ZVerifying data now
112242023-12-10T18:54:21.647ZNo change after verify
112252023-12-10T18:54:21.647ZLoop: 45 Downstairs dump after verify (and repair):
112262023-12-10T18:54:21.669ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
112272023-12-10T18:54:21.669Z 0 000-019 90 90 90 162 162 162 F F F
112282023-12-10T18:54:21.673Z 1 020-039 90 90 90 162 162 162 F F F
112292023-12-10T18:54:21.674Z 2 040-059 82 82 82 146 146 146 F F F
112302023-12-10T18:54:21.674Z 3 060-079 91 91 91 163 163 163 F F F
112312023-12-10T18:54:21.674Z 4 080-099 88 88 88 151 151 151 F F F
112322023-12-10T18:54:21.674Z 5 100-119 85 85 85 149 149 149 F F F
112332023-12-10T18:54:21.674Z 6 120-139 91 91 91 163 163 163 F F F
112342023-12-10T18:54:21.674Z 7 140-159 88 88 88 155 155 155 F F F
112352023-12-10T18:54:21.675Z 8 160-179 88 88 88 152 152 152 F F F
112362023-12-10T18:54:21.677Z 9 180-199 89 89 89 156 156 156 F F F
112372023-12-10T18:54:21.677Z 10 200-219 90 90 90 161 161 161 F F F
112382023-12-10T18:54:21.677Z 11 220-239 90 90 90 160 160 160 F F F
112392023-12-10T18:54:21.680Z 12 240-259 90 90 90 160 160 160 F F F
112402023-12-10T18:54:21.680Z 13 260-279 88 88 88 151 151 151 F F F
112412023-12-10T18:54:21.680Z 14 280-299 86 86 86 150 150 150 F F F
112422023-12-10T18:54:21.680Z 15 300-319 91 91 91 163 163 163 F F F
112432023-12-10T18:54:21.680Z 16 320-339 91 91 91 163 163 163 F F F
112442023-12-10T18:54:21.680Z 17 340-359 85 85 85 149 149 149 F F F
112452023-12-10T18:54:21.680Z 18 360-379 91 91 91 163 163 163 F F F
112462023-12-10T18:54:21.681Z 19 380-399 90 90 90 158 158 158 F F F
112472023-12-10T18:54:21.681Z 20 400-419 90 90 90 161 161 161 F F F
112482023-12-10T18:54:21.681Z 21 420-439 89 89 89 156 156 156 F F F
112492023-12-10T18:54:21.681Z 22 440-459 86 86 86 150 150 150 F F F
112502023-12-10T18:54:21.681Z 23 460-479 82 82 82 145 145 145 F F F
112512023-12-10T18:54:21.681Z 24 480-499 86 86 86 150 150 150 F F F
112522023-12-10T18:54:21.681Z 25 500-519 84 84 84 148 148 148 F F F
112532023-12-10T18:54:21.681Z 26 520-539 82 82 82 143 143 143 F F F
112542023-12-10T18:54:21.682Z 27 540-559 91 91 91 163 163 163 F F F
112552023-12-10T18:54:21.682Z 28 560-579 90 90 90 157 157 157 F F F
112562023-12-10T18:54:21.682Z 29 580-599 90 90 90 157 157 157 F F F
112572023-12-10T18:54:21.682ZMax gen: 91, Max flush: 163
112582023-12-10T18:54:21.685Z
112592023-12-10T18:54:21.685ZBegin loop 46
112602023-12-10T18:54:21.686ZDownstairs to restart: 1
112612023-12-10T18:54:21.703Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.708189683Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530}
112622023-12-10T18:54:21.704Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.708661419Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530}
112632023-12-10T18:54:21.711Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.708726188Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530}
112642023-12-10T18:54:21.711Z{"msg":"Crucible 9b2f5e4c-ccb7-4266-98ca-acc5bd2ad269 has session id: 55a08796-b591-4ee5-a32f-fc5b55df47d8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.708791894Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112652023-12-10T18:54:21.711Z{"msg":"Crucible runtime is spawned
112662023-12-10T18:54:21.711ZUpstairs opts: Upstairs UUID: 9b2f5e4c-ccb7-4266-98ca-acc5bd2ad269, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.709051522Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112672023-12-10T18:54:21.711Z{"msg":"Crucible stats registered with UUID: 9b2f5e4c-ccb7-4266-98ca-acc5bd2ad269","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.709097303Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112682023-12-10T18:54:21.711Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.709135698Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8","task":"up_listen"}
112692023-12-10T18:54:21.711Z{"msg":"The guest has requested activation with gen:92
112702023-12-10T18:54:21.711ZWait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.709249487Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112712023-12-10T18:54:21.712Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.709315099Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"looper":"0","session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112722023-12-10T18:54:21.712Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.709362501Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112732023-12-10T18:54:21.712Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.70943727Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"looper":"2","session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112742023-12-10T18:54:21.712Z{"msg":"[0] 9b2f5e4c-ccb7-4266-98ca-acc5bd2ad269 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.709550665Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"looper":"0","session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112752023-12-10T18:54:21.712Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.709621208Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"looper":"1","session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112762023-12-10T18:54:21.712Z{"msg":"[2] 9b2f5e4c-ccb7-4266-98ca-acc5bd2ad269 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.709708854Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"looper":"2","session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112772023-12-10T18:54:21.712Z{"msg":"[1] 9b2f5e4c-ccb7-4266-98ca-acc5bd2ad269 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.709788587Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"looper":"1","session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112782023-12-10T18:54:21.712Z{"msg":"9b2f5e4c-ccb7-4266-98ca-acc5bd2ad269 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.710205826Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112792023-12-10T18:54:21.712Z{"msg":"Set desired generation to :92","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.71027819Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112802023-12-10T18:54:21.713Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.710382131Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112812023-12-10T18:54:21.713Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.710769811Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112822023-12-10T18:54:21.713Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.710921575Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112832023-12-10T18:54:21.713Z{"msg":"[2] 9b2f5e4c-ccb7-4266-98ca-acc5bd2ad269 (55a08796-b591-4ee5-a32f-fc5b55df47d8) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.711238474Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112842023-12-10T18:54:21.725Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.711309716Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112852023-12-10T18:54:21.725Z{"msg":"[2] client is_active_req TRUE, promote! session 55a08796-b591-4ee5-a32f-fc5b55df47d8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.71145519Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112862023-12-10T18:54:21.725Z{"msg":"[0] 9b2f5e4c-ccb7-4266-98ca-acc5bd2ad269 (55a08796-b591-4ee5-a32f-fc5b55df47d8) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.711526695Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112872023-12-10T18:54:21.725Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.711562604Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112882023-12-10T18:54:21.726Z{"msg":"[0] client is_active_req TRUE, promote! session 55a08796-b591-4ee5-a32f-fc5b55df47d8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.711585664Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112892023-12-10T18:54:21.726Z{"msg":"[1] 9b2f5e4c-ccb7-4266-98ca-acc5bd2ad269 (55a08796-b591-4ee5-a32f-fc5b55df47d8) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.711774933Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112902023-12-10T18:54:21.726Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.711866404Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112912023-12-10T18:54:21.726Z{"msg":"[1] client is_active_req TRUE, promote! session 55a08796-b591-4ee5-a32f-fc5b55df47d8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.711934821Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112922023-12-10T18:54:21.726Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.712025926Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112932023-12-10T18:54:21.726Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.71209099Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112942023-12-10T18:54:21.728Z{"msg":"9b2f5e4c-ccb7-4266-98ca-acc5bd2ad269 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.712117436Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112952023-12-10T18:54:21.728Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.712227294Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112962023-12-10T18:54:21.728Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.712339974Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112972023-12-10T18:54:21.728Z{"msg":"9b2f5e4c-ccb7-4266-98ca-acc5bd2ad269 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.712394984Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112982023-12-10T18:54:21.728Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.712712858Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
112992023-12-10T18:54:21.728Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.712794079Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113002023-12-10T18:54:21.728Z{"msg":"9b2f5e4c-ccb7-4266-98ca-acc5bd2ad269 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.712837762Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113012023-12-10T18:54:21.728Z{"msg":"[2] 9b2f5e4c-ccb7-4266-98ca-acc5bd2ad269 (55a08796-b591-4ee5-a32f-fc5b55df47d8) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.713032483Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113022023-12-10T18:54:21.728Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.713101122Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113032023-12-10T18:54:21.729Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:21.713141718Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113042023-12-10T18:54:21.729Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.713177397Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113052023-12-10T18:54:21.729Z{"msg":"[0] 9b2f5e4c-ccb7-4266-98ca-acc5bd2ad269 (55a08796-b591-4ee5-a32f-fc5b55df47d8) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.713249067Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113062023-12-10T18:54:21.729Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.713361432Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113072023-12-10T18:54:21.729Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:21.713438607Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113082023-12-10T18:54:21.729Z{"msg":"9b2f5e4c-ccb7-4266-98ca-acc5bd2ad269 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.713487997Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113092023-12-10T18:54:21.729Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.713528379Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113102023-12-10T18:54:21.729Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.713569863Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113112023-12-10T18:54:21.729Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.713606185Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113122023-12-10T18:54:21.729Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.71362892Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113132023-12-10T18:54:21.729Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.713658172Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113142023-12-10T18:54:21.729Z{"msg":"9b2f5e4c-ccb7-4266-98ca-acc5bd2ad269 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.713730222Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113152023-12-10T18:54:21.729Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.713767213Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113162023-12-10T18:54:21.729Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.713795959Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113172023-12-10T18:54:21.730Z{"msg":"[1] 9b2f5e4c-ccb7-4266-98ca-acc5bd2ad269 (55a08796-b591-4ee5-a32f-fc5b55df47d8) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.713856091Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113182023-12-10T18:54:21.730Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.713881901Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113192023-12-10T18:54:21.730Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:21.71393886Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113202023-12-10T18:54:21.730Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.713964558Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113212023-12-10T18:54:21.730Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:92
113222023-12-10T18:54:21.730ZWait for a query_work_queue command to finish before sending IO
113232023-12-10T18:54:21.730Z,"time":"2023-12-10T18:54:21.713992747Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113242023-12-10T18:54:21.730Z{"msg":"9b2f5e4c-ccb7-4266-98ca-acc5bd2ad269 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.714081207Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113252023-12-10T18:54:21.730Z{"msg":"[0]R flush_numbers[0..12]: [162, 162, 146, 163, 151, 149, 163, 155, 152, 156, 161, 160]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.714141669Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113262023-12-10T18:54:21.731Z{"msg":"[0]R generation[0..12]: [90, 90, 82, 91, 88, 85, 91, 88, 88, 89, 90, 90]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.714237845Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113272023-12-10T18:54:21.735Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.714268545Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113282023-12-10T18:54:21.736Z{"msg":"[1]R flush_numbers[0..12]: [162, 162, 146, 163, 151, 149, 163, 155, 152, 156, 161, 160]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.714337881Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113292023-12-10T18:54:21.736Z{"msg":"[1]R generation[0..12]: [90, 90, 82, 91, 88, 85, 91, 88, 88, 89, 90, 90]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.714366206Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113302023-12-10T18:54:21.736Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.714449465Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113312023-12-10T18:54:21.736Z{"msg":"[2]R flush_numbers[0..12]: [162, 162, 146, 163, 151, 149, 163, 155, 152, 156, 161, 160]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.714532606Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113322023-12-10T18:54:21.736Z{"msg":"[2]R generation[0..12]: [90, 90, 82, 91, 88, 85, 91, 88, 88, 89, 90, 90]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.714566348Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113332023-12-10T18:54:21.736Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.714605478Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113342023-12-10T18:54:21.736Z{"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.714666316Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113352023-12-10T18:54:21.737Z{"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.714690595Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113362023-12-10T18:54:21.737Z{"msg":"Next flush: 164","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.714712064Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113372023-12-10T18:54:21.737Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.714757746Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113382023-12-10T18:54:21.737Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.714795272Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113392023-12-10T18:54:21.737Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.714859595Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113402023-12-10T18:54:21.737Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.71492783Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113412023-12-10T18:54:21.737Z{"msg":"9b2f5e4c-ccb7-4266-98ca-acc5bd2ad269 is now active with session: 55a08796-b591-4ee5-a32f-fc5b55df47d8","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.714997771Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113422023-12-10T18:54:21.737Z{"msg":"9b2f5e4c-ccb7-4266-98ca-acc5bd2ad269 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.71503968Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113432023-12-10T18:54:21.738Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.715120511Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113442023-12-10T18:54:21.738Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.715188408Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113452023-12-10T18:54:21.738Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.715214Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113462023-12-10T18:54:21.739Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.715234803Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113472023-12-10T18:54:21.739Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.71530235Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113482023-12-10T18:54:21.739Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.715324867Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113492023-12-10T18:54:21.739Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.715344625Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113502023-12-10T18:54:21.739Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.715364552Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113512023-12-10T18:54:21.739Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.715383699Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113522023-12-10T18:54:21.739Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.715403788Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_idRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
113532023-12-10T18:54:21.739Z":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113542023-12-10T18:54:21.739Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:21.715500562Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1530,"session_id":"55a08796-b591-4ee5-a32f-fc5b55df47d8"}
113552023-12-10T18:54:21.739ZLoading write count information from file "/tmp/test_repair_verify.data"
113562023-12-10T18:54:21.739ZRun Repair workload
113572023-12-10T18:54:21.739Z01/30 Write block 529 len 16384 data: 6 7 10 10
113582023-12-10T18:54:21.743Z02/30 Read block 312 len 36864
113592023-12-10T18:54:21.743Z03/30 Flush
113602023-12-10T18:54:21.743Z04/30 Write block 185 len 12288 data: 6 5 5
113612023-12-10T18:54:21.743Z05/30 Write block 550 len 20480 data: 6 5 5 5 5
113622023-12-10T18:54:21.743Z06/30 Write block 218 len 20480 data: 10 9 9 12 12
113632023-12-10T18:54:21.743Z07/30 Write block 190 len 28672 data: 7 8 8 9 9 13 13
113642023-12-10T18:54:21.743Z08/30 Write block 154 len 16384 data: 9 5 5 8
113652023-12-10T18:54:21.743Z09/30 Write block 223 len 4096 data: 10
113662023-12-10T18:54:21.743Z10/30 Write block 501 len 12288 data: 10 10 11
113672023-12-10T18:54:21.743Z11/30 Read block 558 len 32768
113682023-12-10T18:54:21.743Z12/30 Flush
113692023-12-10T18:54:21.743Z13/30 Write block 586 len 4096 data: 5
113702023-12-10T18:54:21.743Z14/30 Read block 299 len 32768
113712023-12-10T18:54:21.750Z15/30 Write block 562 len 36864 data: 9 8 9 9 11 10 8 9 9
113722023-12-10T18:54:21.750Z16/30 Read block 157 len 12288
113732023-12-10T18:54:21.750Z17/30 Read block 44 len 8192
113742023-12-10T18:54:21.750Z18/30 Read block 333 len 12288
113752023-12-10T18:54:21.751Z19/30 Read block 509 len 36864
113762023-12-10T18:54:21.751Z20/30 Write block 535 len 16384 data: 12 7 6 7
113772023-12-10T18:54:21.751Z21/30 Read block 499 len 40960
113782023-12-10T18:54:21.755Z22/30 Write block 248 len 16384 data: 11 12 10 9
113792023-12-10T18:54:21.755Z23/30 Read block 487 len 8192
113802023-12-10T18:54:21.755Z24/30 Flush
113812023-12-10T18:54:21.755Z25/30 Write block 421 len 4096 data: 8
113822023-12-10T18:54:21.755Z26/30 Write block 531 len 8192 data: 11 11
113832023-12-10T18:54:21.759Z27/30 Read block 529 len 32768
113842023-12-10T18:54:21.759Z28/30 Read block 211 len 32768
113852023-12-10T18:54:21.762Z29/30 Read block 16 len 28672
113862023-12-10T18:54:21.766Z30/30 Read block 269 len 32768
113872023-12-10T18:54:21.766Z----------------------------------------------------------------
113882023-12-10T18:54:21.767Z Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30
113892023-12-10T18:54:21.767ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
113902023-12-10T18:54:21.767Z 1 Acked 1000 Write 4 Done Done Done false
113912023-12-10T18:54:21.767Z 2 Acked 1001 Read 9 Done Sent Done false
113922023-12-10T18:54:21.767Z 3 Acked 1002 Flush 0 Done Sent Done false
113932023-12-10T18:54:21.767Z 4 Acked 1003 Write 3 Done Sent Done false
113942023-12-10T18:54:21.767Z 5 Acked 1004 Write 5 Done Sent Done false
113952023-12-10T18:54:21.767Z 6 Acked 1005 Write 5 Done Sent Done false
113962023-12-10T18:54:21.767Z 7 Acked 1006 Write 7 Done Sent Done false
113972023-12-10T18:54:21.768Z 8 Acked 1007 Write 4 Done Sent Done false
113982023-12-10T18:54:21.768Z 9 Acked 1008 Write 1 Done Sent Done false
113992023-12-10T18:54:21.768Z 10 Acked 1009 Write 3 Done Sent Done false
114002023-12-10T18:54:21.768Z 11 Acked 1010 Read 8 Done Sent Done false
114012023-12-10T18:54:21.769Z 12 Acked 1011 Flush 0 Done Sent Done false
114022023-12-10T18:54:21.769Z 13 Acked 1012 Write 1 Done Sent Done false
114032023-12-10T18:54:21.769Z 14 Acked 1013 Read 8 Done Sent Done false
114042023-12-10T18:54:21.769Z 15 Acked 1014 Write 9 Done Sent Done false
114052023-12-10T18:54:21.769Z 16 Acked 1015 Read 3 Done Sent Done false
114062023-12-10T18:54:21.769Z 17 Acked 1016 Read 2 Done Sent Done false
114072023-12-10T18:54:21.769Z 18 Acked 1017 Read 3 Done Sent Done false
114082023-12-10T18:54:21.769Z 19 Acked 1018 Read 9 Done Sent Done false
114092023-12-10T18:54:21.769Z 20 Acked 1019 Write 4 Done Sent Done false
114102023-12-10T18:54:21.769Z 21 Acked 1020 Read 10 Done Sent Done false
114112023-12-10T18:54:21.769Z 22 Acked 1021 Write 4 Done Sent Done false
114122023-12-10T18:54:21.769Z 23 Acked 1022 Read 2 Done Sent Done false
114132023-12-10T18:54:21.769Z 24 Acked 1023 Flush 0 Done Sent Done false
114142023-12-10T18:54:21.769Z 25 Acked 1024 Write 1 Done Sent Done false
114152023-12-10T18:54:21.770Z 26 Acked 1025 Write 2 Done Sent Done false
114162023-12-10T18:54:21.770Z 27 Acked 1026 Read 8 Done Sent Done false
114172023-12-10T18:54:21.770Z 28 Acked 1027 Read 8 Done Sent Done false
114182023-12-10T18:54:21.770Z 29 Acked 1028 Read 7 Done Sent Done false
114192023-12-10T18:54:21.770Z 30 Acked 1029 Read 8 Done Sent Sent false
114202023-12-10T18:54:21.770Z STATES DS:0 DS:1 DS:2 TOTAL
114212023-12-10T18:54:21.770Z New 0 0 0 0
114222023-12-10T18:54:21.770Z Sent 0 29 1 30
114232023-12-10T18:54:21.770Z Done 30 1 29 60
114242023-12-10T18:54:21.770Z Skipped 0 0 0 0
114252023-12-10T18:54:21.770Z Error 0 0 0 0
114262023-12-10T18:54:21.778ZLast Flush: 1023 0 1023
114272023-12-10T18:54:21.778ZDownstairs last five completed:
114282023-12-10T18:54:21.778ZUpstairs last five completed: 30 29 28 27 26 25
114292023-12-10T18:54:21.778ZWrote out file "/tmp/test_repair_verify.data"
114302023-12-10T18:54:21.778Z
114312023-12-10T18:54:21.782ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
114322023-12-10T18:54:21.800ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
114332023-12-10T18:54:21.801Z 0 000-019 90 90 90 162 162 162 F F F
114342023-12-10T18:54:21.801Z 1 020-039 90 90 90 162 162 162 F F F
114352023-12-10T18:54:21.802Z 2 040-059 82 82 82 146 146 146 F F F
114362023-12-10T18:54:21.806Z 3 060-079 91 91 91 163 163 163 F F F
114372023-12-10T18:54:21.806Z 4 080-099 88 88 88 151 151 151 F F F
114382023-12-10T18:54:21.806Z 5 100-119 85 85 85 149 149 149 F F F
114392023-12-10T18:54:21.806Z 6 120-139 91 91 91 163 163 163 F F F
114402023-12-10T18:54:21.806Z 7 140-159 92 88 92 165 155 165 F F F <---
114412023-12-10T18:54:21.806Z 8 160-179 88 88 88 152 152 152 F F F
114422023-12-10T18:54:21.807Z 9 180-199 92 89 92 165 156 165 F F F <---
114432023-12-10T18:54:21.807Z 10 200-219 92 90 92 165 161 165 F F F <---
114442023-12-10T18:54:21.807Z 11 220-239 92 90 92 165 160 165 F F F <---
114452023-12-10T18:54:21.807Z 12 240-259 92 90 92 166 160 166 F F F <---
114462023-12-10T18:54:21.807Z 13 260-279 88 88 88 151 151 151 F F F
114472023-12-10T18:54:21.807Z 14 280-299 86 86 86 150 150 150 F F F
114482023-12-10T18:54:21.807Z 15 300-319 91 91 91 163 163 163 F F F
114492023-12-10T18:54:21.807Z 16 320-339 91 91 91 163 163 163 F F F
114502023-12-10T18:54:21.807Z 17 340-359 85 85 85 149 149 149 F F F
114512023-12-10T18:54:21.808Z 18 360-379 91 91 91 163 163 163 F F F
114522023-12-10T18:54:21.808Z 19 380-399 90 90 90 158 158 158 F F F
114532023-12-10T18:54:21.808Z 20 400-419 90 90 90 161 161 161 F F F
114542023-12-10T18:54:21.808Z 21 420-439 89 89 89 156 156 156 T F T <---
114552023-12-10T18:54:21.808Z 22 440-459 86 86 86 150 150 150 F F F
114562023-12-10T18:54:21.808Z 23 460-479 82 82 82 145 145 145 F F F
114572023-12-10T18:54:21.808Z 24 480-499 86 86 86 150 150 150 F F F
114582023-12-10T18:54:21.808Z 25 500-519 92 84 92 165 148 165 F F F <---
114592023-12-10T18:54:21.809Z 26 520-539 92 82 92 166 143 166 T T T <---
114602023-12-10T18:54:21.809Z 27 540-559 92 91 92 165 163 165 F F F <---
114612023-12-10T18:54:21.809Z 28 560-579 92 90 92 166 157 166 F F F <---
114622023-12-10T18:54:21.809Z 29 580-599 92 90 92 166 157 166 F F F <---
114632023-12-10T18:54:21.809ZMax gen: 92, Max flush: 166
114642023-12-10T18:54:21.809ZError: Difference in extent metadata found!
114652023-12-10T18:54:21.809ZA Difference in extent metadata is expected here
114662023-12-10T18:54:21.809ZOn loop 46
114672023-12-10T18:54:21.809Z
114682023-12-10T18:54:21.813ZVerifying data now
114692023-12-10T18:54:26.916ZNo change after verify
114702023-12-10T18:54:26.916ZLoop: 46 Downstairs dump after verify (and repair):
114712023-12-10T18:54:26.939ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
114722023-12-10T18:54:26.940Z 0 000-019 90 90 90 162 162 162 F F F
114732023-12-10T18:54:26.944Z 1 020-039 90 90 90 162 162 162 F F F
114742023-12-10T18:54:26.944Z 2 040-059 82 82 82 146 146 146 F F F
114752023-12-10T18:54:26.945Z 3 060-079 91 91 91 163 163 163 F F F
114762023-12-10T18:54:26.945Z 4 080-099 88 88 88 151 151 151 F F F
114772023-12-10T18:54:26.945Z 5 100-119 85 85 85 149 149 149 F F F
114782023-12-10T18:54:26.945Z 6 120-139 91 91 91 163 163 163 F F F
114792023-12-10T18:54:26.945Z 7 140-159 92 92 92 165 165 165 F F F
114802023-12-10T18:54:26.945Z 8 160-179 88 88 88 152 152 152 F F F
114812023-12-10T18:54:26.945Z 9 180-199 92 92 92 165 165 165 F F F
114822023-12-10T18:54:26.946Z 10 200-219 92 92 92 165 165 165 F F F
114832023-12-10T18:54:26.946Z 11 220-239 92 92 92 165 165 165 F F F
114842023-12-10T18:54:26.946Z 12 240-259 92 92 92 166 166 166 F F F
114852023-12-10T18:54:26.946Z 13 260-279 88 88 88 151 151 151 F F F
114862023-12-10T18:54:26.947Z 14 280-299 86 86 86 150 150 150 F F F
114872023-12-10T18:54:26.947Z 15 300-319 91 91 91 163 163 163 F F F
114882023-12-10T18:54:26.948Z 16 320-339 91 91 91 163 163 163 F F F
114892023-12-10T18:54:26.948Z 17 340-359 85 85 85 149 149 149 F F F
114902023-12-10T18:54:26.948Z 18 360-379 91 91 91 163 163 163 F F F
114912023-12-10T18:54:26.949Z 19 380-399 90 90 90 158 158 158 F F F
114922023-12-10T18:54:26.949Z 20 400-419 90 90 90 161 161 161 F F F
114932023-12-10T18:54:26.950Z 21 420-439 93 93 93 167 167 167 F F F
114942023-12-10T18:54:26.950Z 22 440-459 86 86 86 150 150 150 F F F
114952023-12-10T18:54:26.950Z 23 460-479 82 82 82 145 145 145 F F F
114962023-12-10T18:54:26.951Z 24 480-499 86 86 86 150 150 150 F F F
114972023-12-10T18:54:26.951Z 25 500-519 92 92 92 165 165 165 F F F
114982023-12-10T18:54:26.951Z 26 520-539 93 93 93 167 167 167 F F F
114992023-12-10T18:54:26.951Z 27 540-559 92 92 92 165 165 165 F F F
115002023-12-10T18:54:26.951Z 28 560-579 92 92 92 166 166 166 F F F
115012023-12-10T18:54:26.951Z 29 580-599 92 92 92 166 166 166 F F F
115022023-12-10T18:54:26.951ZMax gen: 93, Max flush: 167
115032023-12-10T18:54:26.956Z
115042023-12-10T18:54:26.956ZBegin loop 47
115052023-12-10T18:54:26.956ZDownstairs to restart: 1
115062023-12-10T18:54:26.974Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.97883305Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538}
115072023-12-10T18:54:26.974Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.979294384Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538}
115082023-12-10T18:54:26.980Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.979361026Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538}
115092023-12-10T18:54:26.981Z{"msg":"Crucible 0bf5f8eb-2b67-4f4d-bdb1-e4e68a8bf1c4 has session id: 94808ff5-684e-49ef-a982-9f9f87eab0cb","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.979464439Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115102023-12-10T18:54:26.981Z{"msg":"Upstairs opts: Upstairs UUID: 0bf5f8eb-2b67-4f4d-bdb1-e4e68a8bf1c4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.979520567Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115112023-12-10T18:54:26.981Z{"msg":"Crucible stats registered with UUID: 0bf5f8eb-2b67-4f4d-bdb1-e4e68a8bf1c4","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.979590124Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115122023-12-10T18:54:26.981ZCrucible runtime is spawned
115132023-12-10T18:54:26.981ZThe guest has requested activation with gen:94
115142023-12-10T18:54:26.982Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.979751363Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb","task":"up_listen"}
115152023-12-10T18:54:26.982Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.979836907Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"looper":"1","session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115162023-12-10T18:54:26.982Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.979872805Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115172023-12-10T18:54:26.982Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.97994948Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115182023-12-10T18:54:26.985Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.980113006Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"looper":"2","session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115192023-12-10T18:54:26.985Z{"msg":"[1] 0bf5f8eb-2b67-4f4d-bdb1-e4e68a8bf1c4 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.980175351Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"looper":"1","session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115202023-12-10T18:54:26.986Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.980323321Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"looper":"0","session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115212023-12-10T18:54:26.986Z{"msg":"[2] 0bf5f8eb-2b67-4f4d-bdb1-e4e68a8bf1c4 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.980419969Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"looper":"2","session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115222023-12-10T18:54:26.986Z{"msg":"[0] 0bf5f8eb-2b67-4f4d-bdb1-e4e68a8bf1c4 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.980540232Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"looper":"0","session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115232023-12-10T18:54:26.986Z{"msg":"0bf5f8eb-2b67-4f4d-bdb1-e4e68a8bf1c4 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.981025012Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115242023-12-10T18:54:26.986Z{"msg":"Set desired generation to :94","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.981104313Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115252023-12-10T18:54:26.986Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.981336762Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115262023-12-10T18:54:26.987Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.981689262Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115272023-12-10T18:54:26.987Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.981756005Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115282023-12-10T18:54:26.987Z{"msg":"[2] 0bf5f8eb-2b67-4f4d-bdb1-e4e68a8bf1c4 (94808ff5-684e-49ef-a982-9f9f87eab0cb) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.982150679Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115292023-12-10T18:54:26.987Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.982228419Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115302023-12-10T18:54:26.987Z{"msg":"[2] client is_active_req TRUE, promote! session 94808ff5-684e-49ef-a982-9f9f87eab0cb","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.982254271Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115312023-12-10T18:54:26.988Z{"msg":"[0] 0bf5f8eb-2b67-4f4d-bdb1-e4e68a8bf1c4 (94808ff5-684e-49ef-a982-9f9f87eab0cb) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.98245466Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115322023-12-10T18:54:26.988Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.982488941Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115332023-12-10T18:54:26.989Z{"msg":"[0] client is_active_req TRUE, promote! session 94808ff5-684e-49ef-a982-9f9f87eab0cb","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.982512137Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115342023-12-10T18:54:26.989Z{"msg":"[1] 0bf5f8eb-2b67-4f4d-bdb1-e4e68a8bf1c4 (94808ff5-684e-49ef-a982-9f9f87eab0cb) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.982717143Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115352023-12-10T18:54:26.989Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.982825942Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115362023-12-10T18:54:26.990Z{"msg":"[1] client is_active_req TRUE, promote! session 94808ff5-684e-49ef-a982-9f9f87eab0cb","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.982873584Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115372023-12-10T18:54:26.990Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.982945758Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115382023-12-10T18:54:26.990Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.982994696Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115392023-12-10T18:54:26.990Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.983052759Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115402023-12-10T18:54:26.990Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.983082162Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115412023-12-10T18:54:26.990Z{"msg":"0bf5f8eb-2b67-4f4d-bdb1-e4e68a8bf1c4 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.983108632Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115422023-12-10T18:54:26.990Z{"msg":"0bf5f8eb-2b67-4f4d-bdb1-e4e68a8bf1c4 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.983197062Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115432023-12-10T18:54:26.991Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.983414763Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115442023-12-10T18:54:26.991Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.98351003Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115452023-12-10T18:54:26.991Z{"msg":"0bf5f8eb-2b67-4f4d-bdb1-e4e68a8bf1c4 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.983590154Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115462023-12-10T18:54:26.998Z{"msg":"[0] 0bf5f8eb-2b67-4f4d-bdb1-e4e68a8bf1c4 (94808ff5-684e-49ef-a982-9f9f87eab0cb) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.983900012Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115472023-12-10T18:54:26.998Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.983979164Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115482023-12-10T18:54:26.998Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:26.984106199Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115492023-12-10T18:54:26.998Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.984177629Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115502023-12-10T18:54:26.998Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.984219256Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115512023-12-10T18:54:26.998Z{"msg":"0bf5f8eb-2b67-4f4d-bdb1-e4e68a8bf1c4 WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.984289575Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115522023-12-10T18:54:26.998Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.984329607Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115532023-12-10T18:54:26.998Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.984412355Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115542023-12-10T18:54:26.998Z{"msg":"[2] 0bf5f8eb-2b67-4f4d-bdb1-e4e68a8bf1c4 (94808ff5-684e-49ef-a982-9f9f87eab0cb) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.984475823Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115552023-12-10T18:54:26.998Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.984501403Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115562023-12-10T18:54:26.998Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:26.98454171Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115572023-12-10T18:54:26.998Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.984580357Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115582023-12-10T18:54:26.998Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.984654689Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115592023-12-10T18:54:26.999Z{"msg":"0bf5f8eb-2b67-4f4d-bdb1-e4e68a8bf1c4 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.984716606Z","The guest has finished waiting for activation with:94
115602023-12-10T18:54:26.999ZWait for a query_work_queue command to finish before sending IO
115612023-12-10T18:54:26.999Zhostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115622023-12-10T18:54:26.999Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.984838227Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115632023-12-10T18:54:26.999Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.984877319Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115642023-12-10T18:54:27.000Z{"msg":"[1] 0bf5f8eb-2b67-4f4d-bdb1-e4e68a8bf1c4 (94808ff5-684e-49ef-a982-9f9f87eab0cb) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.98494542Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115652023-12-10T18:54:27.000Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.984972759Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115662023-12-10T18:54:27.000Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:26.984993083Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115672023-12-10T18:54:27.000Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.985056438Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115682023-12-10T18:54:27.000Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.98509918Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115692023-12-10T18:54:27.000Z{"msg":"0bf5f8eb-2b67-4f4d-bdb1-e4e68a8bf1c4 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.985168569Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115702023-12-10T18:54:27.000Z{"msg":"[0]R flush_numbers[0..12]: [162, 162, 146, 163, 151, 149, 163, 165, 152, 165, 165, 165]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.985203891Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115712023-12-10T18:54:27.000Z{"msg":"[0]R generation[0..12]: [90, 90, 82, 91, 88, 85, 91, 92, 88, 92, 92, 92]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.985227916Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115722023-12-10T18:54:27.000Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.985286944Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115732023-12-10T18:54:27.000Z{"msg":"[1]R flush_numbers[0..12]: [162, 162, 146, 163, 151, 149, 163, 165, 152, 165, 165, 165]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.985312437Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115742023-12-10T18:54:27.000Z{"msg":"[1]R generation[0..12]: [90, 90, 82, 91, 88, 85, 91, 92, 88, 92, 92, 92]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.985378601Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115752023-12-10T18:54:27.000Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.985422058Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115762023-12-10T18:54:27.000Z{"msg":"[2]R flush_numbers[0..12]: [162, 162, 146, 163, 151, 149, 163, 165, 152, 165, 165, 165]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.985484142Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115772023-12-10T18:54:27.001Z{"msg":"[2]R generation[0..12]: [90, 90, 82, 91, 88, 85, 91, 92, 88, 92, 92, 92]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.985508182Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115782023-12-10T18:54:27.001Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.985567059Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115792023-12-10T18:54:27.001Z{"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.98563381Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115802023-12-10T18:54:27.001Z{"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.985659638Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115812023-12-10T18:54:27.001Z{"msg":"Next flush: 168","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.985702541Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115822023-12-10T18:54:27.002Z{"msg":"All extents match","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
115832023-12-10T18:54:27.002Z,"time":"2023-12-10T18:54:26.985736406Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115842023-12-10T18:54:27.002Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.985871288Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115852023-12-10T18:54:27.002Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.985943311Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115862023-12-10T18:54:27.002Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.986012661Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115872023-12-10T18:54:27.002Z{"msg":"0bf5f8eb-2b67-4f4d-bdb1-e4e68a8bf1c4 is now active with session: 94808ff5-684e-49ef-a982-9f9f87eab0cb","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.986081384Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115882023-12-10T18:54:27.002Z{"msg":"0bf5f8eb-2b67-4f4d-bdb1-e4e68a8bf1c4 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.986147507Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115892023-12-10T18:54:27.002Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.986183843Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115902023-12-10T18:54:27.002Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.986216548Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115912023-12-10T18:54:27.002Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.986279151Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115922023-12-10T18:54:27.002Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.986316798Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115932023-12-10T18:54:27.002Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.986378834Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115942023-12-10T18:54:27.002Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.986442604Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115952023-12-10T18:54:27.002Z{"msg":"Loading write count information from file "/tmp/test_repair_verify.data"
115962023-12-10T18:54:27.002Z[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.986516322ZRun Repair workload
115972023-12-10T18:54:27.002Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115982023-12-10T18:54:27.003Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.986557796Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
115992023-12-10T18:54:27.003Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.986625118Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
116002023-12-10T18:54:27.003Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.986690887Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
116012023-12-10T18:54:27.003Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:26.986757946Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1538,"session_id":"94808ff5-684e-49ef-a982-9f9f87eab0cb"}
116022023-12-10T18:54:27.003Z01/30 Write block 570 len 16384 data: 10 7 5 6
116032023-12-10T18:54:27.003Z02/30 Write block 284 len 40960 data: 9 11 11 11 10 10 7 9 8 7
116042023-12-10T18:54:27.003Z03/30 Read block 513 len 36864
116052023-12-10T18:54:27.003Z04/30 Write block 126 len 12288 data: 11 10 11
116062023-12-10T18:54:27.003Z05/30 Write block 424 len 16384 data: 10 7 6 7
116072023-12-10T18:54:27.003Z06/30 Write block 214 len 20480 data: 9 10 12 12 11
116082023-12-10T18:54:27.003Z07/30 Read block 440 len 20480
116092023-12-10T18:54:27.003Z08/30 Write block 273 len 32768 data: 8 9 11 10 10 11 8 7
116102023-12-10T18:54:27.003Z09/30 Read block 61 len 32768
116112023-12-10T18:54:27.007Z10/30 Flush
116122023-12-10T18:54:27.008Z11/30 Write block 251 len 32768 data: 10 9 10 7 6 7 3 3
116132023-12-10T18:54:27.008Z12/30 Read block 542 len 24576
116142023-12-10T18:54:27.008Z13/30 Flush
116152023-12-10T18:54:27.009Z14/30 Write block 230 len 8192 data: 11 10
116162023-12-10T18:54:27.009Z15/30 Write block 252 len 28672 data: 10 11 8 7 8 4 4
116172023-12-10T18:54:27.012Z16/30 Write block 84 len 8192 data: 9 11
116182023-12-10T18:54:27.013Z17/30 Flush
116192023-12-10T18:54:27.013Z18/30 Write block 46 len 24576 data: 5 6 6 7 7 7
116202023-12-10T18:54:27.017Z19/30 Read block 269 len 12288
116212023-12-10T18:54:27.017Z20/30 Write block 567 len 24576 data: 11 9 10 11 8 6
116222023-12-10T18:54:27.017Z21/30 Flush
116232023-12-10T18:54:27.021Z22/30 Flush
116242023-12-10T18:54:27.021Z23/30 Write block 427 len 24576 data: 8 6 5 5 4 3
116252023-12-10T18:54:27.022Z24/30 Write block 351 len 36864 data: 8 9 11 9 7 8 7 7 7
116262023-12-10T18:54:27.026Z25/30 Write block 224 len 40960 data: 10 8 10 10 9 9 12 11 10 7
116272023-12-10T18:54:27.026Z26/30 Write block 427 len 32768 data: 9 7 6 6 5 4 4 4
116282023-12-10T18:54:27.026Z27/30 Read block 321 len 20480
116292023-12-10T18:54:27.029Z28/30 Write block 428 len 24576 data: 8 7 7 6 5 5
116302023-12-10T18:54:27.030Z29/30 Write block 88 len 20480 data: 10 8 7 6 5
116312023-12-10T18:54:27.030Z30/30 Write block 444 len 20480 data: 5 7 6 6 7
116322023-12-10T18:54:27.035Z----------------------------------------------------------------
116332023-12-10T18:54:27.035Z Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:30
116342023-12-10T18:54:27.035ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
116352023-12-10T18:54:27.035Z 1 Acked 1000 Write 4 Done Sent Done false
116362023-12-10T18:54:27.035Z 2 Acked 1001 Write 10 Done Sent Done false
116372023-12-10T18:54:27.035Z 3 Acked 1002 Read 9 Done Sent Done false
116382023-12-10T18:54:27.036Z 4 Acked 1003 Write 3 Done Sent Done false
116392023-12-10T18:54:27.036Z 5 Acked 1004 Write 4 Done Sent Done false
116402023-12-10T18:54:27.036Z 6 Acked 1005 Write 5 Done Sent Done false
116412023-12-10T18:54:27.036Z 7 Acked 1006 Read 5 Done Sent Done false
116422023-12-10T18:54:27.036Z 8 Acked 1007 Write 8 Done Sent Done false
116432023-12-10T18:54:27.037Z 9 Acked 1008 Read 8 Done Sent Done false
116442023-12-10T18:54:27.037Z 10 Acked 1009 Flush 0 Done Sent Done false
116452023-12-10T18:54:27.037Z 11 Acked 1010 Write 8 Done Sent Done false
116462023-12-10T18:54:27.037Z 12 Acked 1011 Read 6 Done Sent Done false
116472023-12-10T18:54:27.037Z 13 Acked 1012 Flush 0 Done Sent Done false
116482023-12-10T18:54:27.038Z 14 Acked 1013 Write 2 Done Sent Done false
116492023-12-10T18:54:27.038Z 15 Acked 1014 Write 7 Done Sent Done false
116502023-12-10T18:54:27.038Z 16 Acked 1015 Write 2 Done Sent Done false
116512023-12-10T18:54:27.038Z 17 Acked 1016 Flush 0 Done Sent Done false
116522023-12-10T18:54:27.038Z 18 Acked 1017 Write 6 Done Sent Done false
116532023-12-10T18:54:27.039Z 19 Acked 1018 Read 3 Done Sent Done false
116542023-12-10T18:54:27.039Z 20 Acked 1019 Write 6 Done Sent Done false
116552023-12-10T18:54:27.039Z 21 Acked 1020 Flush 0 Done Sent Done false
116562023-12-10T18:54:27.039Z 22 Acked 1021 Flush 0 Done Sent Done false
116572023-12-10T18:54:27.040Z 23 Acked 1022 Write 6 Done Sent Done false
116582023-12-10T18:54:27.040Z 24 Acked 1023 Write 9 Done Sent Done false
116592023-12-10T18:54:27.040Z 25 Acked 1024 Write 10 Done Sent Done false
116602023-12-10T18:54:27.040Z 26 Acked 1025 Write 8 Done Sent Done false
116612023-12-10T18:54:27.041Z 27 Acked 1026 Read 5 Done Sent Done false
116622023-12-10T18:54:27.041Z 28 Acked 1027 Write 6 Sent Sent Sent false
116632023-12-10T18:54:27.041Z 29 Acked 1028 Write 5 Sent Sent Sent false
116642023-12-10T18:54:27.041Z 30 Acked 1029 Write 5 Sent Sent Sent false
116652023-12-10T18:54:27.041Z STATES DS:0 DS:1 DS:2 TOTAL
116662023-12-10T18:54:27.041Z New 0 0 0 0
116672023-12-10T18:54:27.042Z Sent 3 30 3 36
116682023-12-10T18:54:27.042Z Done 27 0 27 54
116692023-12-10T18:54:27.042Z Skipped 0 0 0 0
116702023-12-10T18:54:27.042Z Error 0 0 0 0
116712023-12-10T18:54:27.042ZLast Flush: 1021 0 1021
116722023-12-10T18:54:27.042ZDownstairs last five completed:
116732023-12-10T18:54:27.043ZUpstairs last five completed: 30 29 28 27 26 25
116742023-12-10T18:54:27.043ZWrote out file "/tmp/test_repair_verify.data"
116752023-12-10T18:54:27.047Z
116762023-12-10T18:54:27.047ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
116772023-12-10T18:54:27.066ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
116782023-12-10T18:54:27.066Z 0 000-019 90 90 90 162 162 162 F F F
116792023-12-10T18:54:27.066Z 1 020-039 90 90 90 162 162 162 F F F
116802023-12-10T18:54:27.066Z 2 040-059 94 82 94 171 146 171 F F F <---
116812023-12-10T18:54:27.066Z 3 060-079 91 91 91 163 163 163 F F F
116822023-12-10T18:54:27.070Z 4 080-099 94 88 94 170 151 170 T F T <---
116832023-12-10T18:54:27.071Z 5 100-119 85 85 85 149 149 149 F F F
116842023-12-10T18:54:27.071Z 6 120-139 94 91 94 168 163 168 F F F <---
116852023-12-10T18:54:27.071Z 7 140-159 92 92 92 165 165 165 F F F
116862023-12-10T18:54:27.071Z 8 160-179 88 88 88 152 152 152 F F F
116872023-12-10T18:54:27.072Z 9 180-199 92 92 92 165 165 165 F F F
116882023-12-10T18:54:27.072Z 10 200-219 94 92 94 168 165 168 F F F <---
116892023-12-10T18:54:27.072Z 11 220-239 94 92 94 170 165 170 T F T <---
116902023-12-10T18:54:27.072Z 12 240-259 94 92 94 170 166 170 F F F <---
116912023-12-10T18:54:27.072Z 13 260-279 94 88 94 168 151 168 F F F <---
116922023-12-10T18:54:27.072Z 14 280-299 94 86 94 168 150 168 F F F <---
116932023-12-10T18:54:27.072Z 15 300-319 91 91 91 163 163 163 F F F
116942023-12-10T18:54:27.072Z 16 320-339 91 91 91 163 163 163 F F F
116952023-12-10T18:54:27.073Z 17 340-359 85 85 85 149 149 149 T F T <---
116962023-12-10T18:54:27.073Z 18 360-379 91 91 91 163 163 163 F F F
116972023-12-10T18:54:27.073Z 19 380-399 90 90 90 158 158 158 F F F
116982023-12-10T18:54:27.073Z 20 400-419 90 90 90 161 161 161 F F F
116992023-12-10T18:54:27.073Z 21 420-439 94 93 94 168 167 168 T F T <---
117002023-12-10T18:54:27.074Z 22 440-459 86 86 86 150 150 150 T F T <---
117012023-12-10T18:54:27.074Z 23 460-479 82 82 82 145 145 145 F F F
117022023-12-10T18:54:27.074Z 24 480-499 86 86 86 150 150 150 F F F
117032023-12-10T18:54:27.075Z 25 500-519 92 92 92 165 165 165 F F F
117042023-12-10T18:54:27.075Z 26 520-539 93 93 93 167 167 167 F F F
117052023-12-10T18:54:27.075Z 27 540-559 92 92 92 165 165 165 F F F
117062023-12-10T18:54:27.075Z 28 560-579 94 92 94 171 166 171 F F F <---
117072023-12-10T18:54:27.075Z 29 580-599 92 92 92 166 166 166 F F F
117082023-12-10T18:54:27.075ZMax gen: 94, Max flush: 171
117092023-12-10T18:54:27.075ZError: Difference in extent metadata found!
117102023-12-10T18:54:27.079ZA Difference in extent metadata is expected here
117112023-12-10T18:54:27.080ZOn loop 47
117122023-12-10T18:54:27.080Z
117132023-12-10T18:54:27.080ZVerifying data now
117142023-12-10T18:54:32.248ZNo change after verify
117152023-12-10T18:54:32.248ZLoop: 47 Downstairs dump after verify (and repair):
117162023-12-10T18:54:32.271ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
117172023-12-10T18:54:32.271Z 0 000-019 90 90 90 162 162 162 F F F
117182023-12-10T18:54:32.274Z 1 020-039 90 90 90 162 162 162 F F F
117192023-12-10T18:54:32.274Z 2 040-059 94 94 94 171 171 171 F F F
117202023-12-10T18:54:32.274Z 3 060-079 91 91 91 163 163 163 F F F
117212023-12-10T18:54:32.274Z 4 080-099 95 95 95 172 172 172 F F F
117222023-12-10T18:54:32.274Z 5 100-119 85 85 85 149 149 149 F F F
117232023-12-10T18:54:32.274Z 6 120-139 94 94 94 168 168 168 F F F
117242023-12-10T18:54:32.275Z 7 140-159 92 92 92 165 165 165 F F F
117252023-12-10T18:54:32.275Z 8 160-179 88 88 88 152 152 152 F F F
117262023-12-10T18:54:32.275Z 9 180-199 92 92 92 165 165 165 F F F
117272023-12-10T18:54:32.275Z 10 200-219 94 94 94 168 168 168 F F F
117282023-12-10T18:54:32.275Z 11 220-239 95 95 95 172 172 172 F F F
117292023-12-10T18:54:32.275Z 12 240-259 94 94 94 170 170 170 F F F
117302023-12-10T18:54:32.275Z 13 260-279 94 94 94 168 168 168 F F F
117312023-12-10T18:54:32.275Z 14 280-299 94 94 94 168 168 168 F F F
117322023-12-10T18:54:32.275Z 15 300-319 91 91 91 163 163 163 F F F
117332023-12-10T18:54:32.275Z 16 320-339 91 91 91 163 163 163 F F F
117342023-12-10T18:54:32.275Z 17 340-359 95 95 95 172 172 172 F F F
117352023-12-10T18:54:32.275Z 18 360-379 91 91 91 163 163 163 F F F
117362023-12-10T18:54:32.275Z 19 380-399 90 90 90 158 158 158 F F F
117372023-12-10T18:54:32.275Z 20 400-419 90 90 90 161 161 161 F F F
117382023-12-10T18:54:32.275Z 21 420-439 95 95 95 172 172 172 F F F
117392023-12-10T18:54:32.276Z 22 440-459 95 95 95 172 172 172 F F F
117402023-12-10T18:54:32.276Z 23 460-479 82 82 82 145 145 145 F F F
117412023-12-10T18:54:32.276Z 24 480-499 86 86 86 150 150 150 F F F
117422023-12-10T18:54:32.276Z 25 500-519 92 92 92 165 165 165 F F F
117432023-12-10T18:54:32.276Z 26 520-539 93 93 93 167 167 167 F F F
117442023-12-10T18:54:32.276Z 27 540-559 92 92 92 165 165 165 F F F
117452023-12-10T18:54:32.276Z 28 560-579 94 94 94 171 171 171 F F F
117462023-12-10T18:54:32.276Z 29 580-599 92 92 92 166 166 166 F F F
117472023-12-10T18:54:32.276ZMax gen: 95, Max flush: 172
117482023-12-10T18:54:32.282Z
117492023-12-10T18:54:32.282ZBegin loop 48
117502023-12-10T18:54:32.282ZDownstairs to restart: 2
117512023-12-10T18:54:32.312Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.316405067Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546}
117522023-12-10T18:54:32.312Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.316826533Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546}
117532023-12-10T18:54:32.317Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.316878589Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546}
117542023-12-10T18:54:32.317Z{"msg":"Crucible 203b94e1-7e18-49a2-8e55-5a314a6b7c99 has session id: 76dbc329-7aec-4d5e-b46b-c21105509783","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.316923419Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117552023-12-10T18:54:32.317Z{"msg":"Upstairs opts: Upstairs UUID: 203b94e1-7e18-49a2-8e55-5a314a6b7c99, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.316961523Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117562023-12-10T18:54:32.318Z{"msg":"Crucible stats registered with UUID: 203b94e1-7e18-49a2-8e55-5a314a6b7c99","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.317008966Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117572023-12-10T18:54:32.318ZCrucible runtime is spawned
117582023-12-10T18:54:32.318Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.317426045Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783","task":"up_listen"}
117592023-12-10T18:54:32.318Z{"msg":"[0] connecting to 127.0.0.1:8810"The guest has requested activation with gen:96
117602023-12-10T18:54:32.318Z,"v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.317564802Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"looper":"0","session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117612023-12-10T18:54:32.318Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.317652691Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117622023-12-10T18:54:32.318Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.317680104Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117632023-12-10T18:54:32.318Z{"msg":"[0] 203b94e1-7e18-49a2-8e55-5a314a6b7c99 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.317997165Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"looper":"0","session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117642023-12-10T18:54:32.318Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.318060527Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"looper":"2","session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117652023-12-10T18:54:32.319Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.318289335Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"looper":"1","session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117662023-12-10T18:54:32.319Z{"msg":"[2] 203b94e1-7e18-49a2-8e55-5a314a6b7c99 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.318375172Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"looper":"2","session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117672023-12-10T18:54:32.319Z{"msg":"[1] 203b94e1-7e18-49a2-8e55-5a314a6b7c99 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.318444211Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"looper":"1","session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117682023-12-10T18:54:32.319Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.318921951Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117692023-12-10T18:54:32.319Z{"msg":"203b94e1-7e18-49a2-8e55-5a314a6b7c99 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.318999293Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117702023-12-10T18:54:32.319Z{"msg":"Set desired generation to :96","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.319036812Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117712023-12-10T18:54:32.319Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.319337194Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117722023-12-10T18:54:32.319Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.319455167Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117732023-12-10T18:54:32.320Z{"msg":"[0] 203b94e1-7e18-49a2-8e55-5a314a6b7c99 (76dbc329-7aec-4d5e-b46b-c21105509783) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.319736185Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117742023-12-10T18:54:32.320Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.31980411Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117752023-12-10T18:54:32.320Z{"msg":"[0] client is_active_req TRUE, promote! session 76dbc329-7aec-4d5e-b46b-c21105509783","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.319830183Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117762023-12-10T18:54:32.320Z{"msg":"[1] 203b94e1-7e18-49a2-8e55-5a314a6b7c99 (76dbc329-7aec-4d5e-b46b-c21105509783) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.319853481Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117772023-12-10T18:54:32.320Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.31989528Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117782023-12-10T18:54:32.320Z{"msg":"[1] client is_active_req TRUE, promote! session 76dbc329-7aec-4d5e-b46b-c21105509783","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.319923608Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117792023-12-10T18:54:32.320Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.320592922Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117802023-12-10T18:54:32.320Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.320668393Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117812023-12-10T18:54:32.321Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.32071874Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117822023-12-10T18:54:32.321Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.320788099Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117832023-12-10T18:54:32.321Z{"msg":"203b94e1-7e18-49a2-8e55-5a314a6b7c99 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.320829405Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117842023-12-10T18:54:32.321Z{"msg":"203b94e1-7e18-49a2-8e55-5a314a6b7c99 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.320882778Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117852023-12-10T18:54:32.321Z{"msg":"[2] 203b94e1-7e18-49a2-8e55-5a314a6b7c99 (76dbc329-7aec-4d5e-b46b-c21105509783) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.321322438Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117862023-12-10T18:54:32.322Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.321460176Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117872023-12-10T18:54:32.429Z{"msg":"[2] client is_active_req TRUE, promote! session 76dbc329-7aec-4d5e-b46b-c21105509783","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.321590508Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117882023-12-10T18:54:32.430Z{"msg":"[0] 203b94e1-7e18-49a2-8e55-5a314a6b7c99 (76dbc329-7aec-4d5e-b46b-c21105509783) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.32171144Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117892023-12-10T18:54:32.430Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.321756348Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117902023-12-10T18:54:32.430Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:32.321796925Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117912023-12-10T18:54:32.430Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.321864941Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117922023-12-10T18:54:32.430Z{"msg":"[1] 203b94e1-7e18-49a2-8e55-5a314a6b7c99 (76dbc329-7aec-4d5e-b46b-c21105509783) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.321906615Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117932023-12-10T18:54:32.430Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.321954268Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117942023-12-10T18:54:32.431Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:32.321977026Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117952023-12-10T18:54:32.431Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.321996977Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117962023-12-10T18:54:32.431Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.322039562Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117972023-12-10T18:54:32.432Z{"msg":"203b94e1-7e18-49a2-8e55-5a314a6b7c99 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.322072642Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117982023-12-10T18:54:32.432Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.322143628Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
117992023-12-10T18:54:32.432Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.322218397Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118002023-12-10T18:54:32.432Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.32226197Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118012023-12-10T18:54:32.432Z{"msg":"203b94e1-7e18-49a2-8e55-5a314a6b7c99 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.322318224Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118022023-12-10T18:54:32.432Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.322342727Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118032023-12-10T18:54:32.432Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.322362427Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118042023-12-10T18:54:32.432Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.322385478Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118052023-12-10T18:54:32.432Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.322411131Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118062023-12-10T18:54:32.432Z{"msg":"203b94e1-7e18-49a2-8e55-5a314a6b7c99 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.322528316Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118072023-12-10T18:54:32.432Z{"msg":"[2] 203b94e1-7e18-49a2-8e55-5a314a6b7c99 (76dbc329-7aec-4d5e-b46b-c21105509783) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.322920591Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118082023-12-10T18:54:32.432Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.323005902Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118092023-12-10T18:54:32.433Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:32.323051682Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118102023-12-10T18:54:32.433Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.323120248Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118112023-12-10T18:54:32.433Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.323189112Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118122023-12-10T18:54:32.433Z{"msg":"203b94e1-7e18-49a2-8e55-5a314a6b7c99 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.323261476Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118132023-12-10T18:54:32.433Z{The guest has finished waiting for activation with:96
118142023-12-10T18:54:32.433Z"msg":"Wait for a query_work_queue command to finish before sending IO
118152023-12-10T18:54:32.433Z[0]R flush_numbers[0..12]: [162, 162, 171, 163, 172, 149, 168, 165, 152, 165, 168, 172]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.323337586Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118162023-12-10T18:54:32.439Z{"msg":"[0]R generation[0..12]: [90, 90, 94, 91, 95, 85, 94, 92, 88, 92, 94, 95]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.323412796Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118172023-12-10T18:54:32.439Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.323494891Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118182023-12-10T18:54:32.439Z{"msg":"[1]R flush_numbers[0..12]: [162, 162, 171, 163, 172, 149, 168, 165, 152, 165, 168, 172]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.323592636Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118192023-12-10T18:54:32.439Z{"msg":"[1]R generation[0..12]: [90, 90, 94, 91, 95, 85, 94, 92, 88, 92, 94, 95]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.323629174Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118202023-12-10T18:54:32.439Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.323747068Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118212023-12-10T18:54:32.439Z{"msg":"[2]R flush_numbers[0..12]: [162, 162, 171, 163, 172, 149, 168, 165, 152, 165, 168, 172]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.32377212Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118222023-12-10T18:54:32.439Z{"msg":"[2]R generation[0..12]: [90, 90, 94, 91, 95, 85, 94, 92, 88, 92, 94, 95]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.323793942Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118232023-12-10T18:54:32.439Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.323859189Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118242023-12-10T18:54:32.439Z{"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.323881686Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118252023-12-10T18:54:32.439Z{"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.323932163Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118262023-12-10T18:54:32.439Z{"msg":"Next flush: 173","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.323955985Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118272023-12-10T18:54:32.440Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.324003802Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118282023-12-10T18:54:32.440Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.324025968Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118292023-12-10T18:54:32.440Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.324046199Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118302023-12-10T18:54:32.440Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.324104325Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118312023-12-10T18:54:32.441Z{"msg":"203b94e1-7e18-49a2-8e55-5a314a6b7c99 is now active with session: 76dbc329-7aec-4d5e-b46b-c21105509783","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.324133185Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118322023-12-10T18:54:32.441Z{"msg":"203b94e1-7e18-49a2-8e55-5a314a6b7c99 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.324154785Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118332023-12-10T18:54:32.441Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.324197553Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118342023-12-10T18:54:32.441Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.324219904Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118352023-12-10T18:54:32.441Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.324239769Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118362023-12-10T18:54:32.441Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.324261361Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118372023-12-10T18:54:32.442Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.324281365Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118382023-12-10T18:54:32.442Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.32430533Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118392023-12-10T18:54:32.442Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.324372144Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118402023-12-10T18:54:32.442Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.32441461Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118412023-12-10T18:54:32.442Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.324481886Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118422023-12-10T18:54:32.443Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.324543375Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118432023-12-10T18:54:32.443Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:32.324568496Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1546,"session_id":"76dbc329-7aec-4d5e-b46b-c21105509783"}
118442023-12-10T18:54:32.443ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
118452023-12-10T18:54:32.443ZLoading write count information from file "/tmp/test_repair_verify.data"
118462023-12-10T18:54:32.443ZRun Repair workload
118472023-12-10T18:54:32.443Z01/30 Write block 343 len 28672 data: 12 11 9 8 9 7 9
118482023-12-10T18:54:32.443Z02/30 Write block 225 len 36864 data: 9 11 11 10 10 13 12 11 8
118492023-12-10T18:54:32.443Z03/30 Read block 104 len 28672
118502023-12-10T18:54:32.448Z04/30 Read block 211 len 28672
118512023-12-10T18:54:32.448Z05/30 Read block 6 len 36864
118522023-12-10T18:54:32.448Z06/30 Read block 293 len 36864
118532023-12-10T18:54:32.449Z07/30 Write block 521 len 12288 data: 6 5 5
118542023-12-10T18:54:32.449Z08/30 Flush
118552023-12-10T18:54:32.449Z09/30 Write block 463 len 32768 data: 5 5 8 7 8 9 11 10
118562023-12-10T18:54:32.449Z10/30 Read block 553 len 16384
118572023-12-10T18:54:32.449Z11/30 Read block 384 len 36864
118582023-12-10T18:54:32.449Z12/30 Read block 337 len 28672
118592023-12-10T18:54:32.449Z13/30 Write block 201 len 36864 data: 11 9 11 9 9 6 6 5 4
118602023-12-10T18:54:32.449Z14/30 Write block 590 len 40960 data: 7 6 6 5 5 4 4 4 2 2
118612023-12-10T18:54:32.450Z15/30 Write block 36 len 20480 data: 10 9 8 9 7
118622023-12-10T18:54:32.450Z16/30 Write block 349 len 16384 data: 10 8 9 10
118632023-12-10T18:54:32.450Z17/30 Read block 591 len 20480
118642023-12-10T18:54:32.450Z18/30 Read block 283 len 24576
118652023-12-10T18:54:32.450Z19/30 Read block 383 len 32768
118662023-12-10T18:54:32.450Z20/30 Read block 463 len 36864
118672023-12-10T18:54:32.450Z21/30 Write block 443 len 32768 data: 5 6 8 7 7 8 7 7
118682023-12-10T18:54:32.450Z22/30 Read block 478 len 32768
118692023-12-10T18:54:32.450Z23/30 Read block 441 len 28672
118702023-12-10T18:54:32.450Z24/30 Write block 462 len 4096 data: 5
118712023-12-10T18:54:32.450Z25/30 Read block 96 len 8192
118722023-12-10T18:54:32.450Z26/30 Flush
118732023-12-10T18:54:32.450Z27/30 Write block 224 len 4096 data: 11
118742023-12-10T18:54:32.450Z28/30 Write block 390 len 16384 data: 8 6 7 5
118752023-12-10T18:54:32.450Z29/30 Read block 94 len 8192
118762023-12-10T18:54:32.451Z30/30 Read block 396 len 8192
118772023-12-10T18:54:32.451Z----------------------------------------------------------------
118782023-12-10T18:54:32.451Z Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:30
118792023-12-10T18:54:32.451ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
118802023-12-10T18:54:32.451Z 1 Acked 1000 Write 7 Done Done Done false
118812023-12-10T18:54:32.451Z 2 Acked 1001 Write 9 Done Done Done false
118822023-12-10T18:54:32.451Z 3 Acked 1002 Read 7 Done Done Done false
118832023-12-10T18:54:32.451Z 4 Acked 1003 Read 7 Done Done Sent false
118842023-12-10T18:54:32.451Z 5 Acked 1004 Read 9 Done Done Sent false
118852023-12-10T18:54:32.451Z 6 Acked 1005 Read 9 Done Done Sent false
118862023-12-10T18:54:32.451Z 7 Acked 1006 Write 3 Done Done Sent false
118872023-12-10T18:54:32.451Z 8 Acked 1007 Flush 0 Done Done Sent false
118882023-12-10T18:54:32.451Z 9 Acked 1008 Write 8 Done Done Sent false
118892023-12-10T18:54:32.451Z 10 Acked 1009 Read 4 Done Done Sent false
118902023-12-10T18:54:32.451Z 11 Acked 1010 Read 9 Done Done Sent false
118912023-12-10T18:54:32.452Z 12 Acked 1011 Read 7 Done Done Sent false
118922023-12-10T18:54:32.452Z 13 Acked 1012 Write 9 Done Done Sent false
118932023-12-10T18:54:32.452Z 14 Acked 1013 Write 10 Done Done Sent false
118942023-12-10T18:54:32.452Z 15 Acked 1014 Write 5 Done Done Sent false
118952023-12-10T18:54:32.452Z 16 Acked 1015 Write 4 Done Done Sent false
118962023-12-10T18:54:32.452Z 17 Acked 1016 Read 5 Done Done Sent false
118972023-12-10T18:54:32.452Z 18 Acked 1017 Read 6 Done Done Sent false
118982023-12-10T18:54:32.452Z 19 Acked 1018 Read 8 Done Done Sent false
118992023-12-10T18:54:32.452Z 20 Acked 1019 Read 9 Done Done Sent false
119002023-12-10T18:54:32.452Z 21 Acked 1020 Write 8 Done Done Sent false
119012023-12-10T18:54:32.452Z 22 Acked 1021 Read 8 Done Done Sent false
119022023-12-10T18:54:32.455Z 23 Acked 1022 Read 7 Done Done Sent false
119032023-12-10T18:54:32.455Z 24 Acked 1023 Write 1 Done Done Sent false
119042023-12-10T18:54:32.455Z 25 Acked 1024 Read 2 Done Done Sent false
119052023-12-10T18:54:32.455Z 26 Acked 1025 Flush 0 Done Done Sent false
119062023-12-10T18:54:32.455Z 27 Acked 1026 Write 1 Done Done Sent false
119072023-12-10T18:54:32.455Z 28 Acked 1027 Write 4 Done Done Sent false
119082023-12-10T18:54:32.455Z 29 Acked 1028 Read 2 Done Done Sent false
119092023-12-10T18:54:32.455Z 30 Acked 1029 Read 2 Done Sent Sent false
119102023-12-10T18:54:32.456Z STATES DS:0 DS:1 DS:2 TOTAL
119112023-12-10T18:54:32.456Z New 0 0 0 0
119122023-12-10T18:54:32.456Z Sent 0 1 27 28
119132023-12-10T18:54:32.456Z Done 30 29 3 62
119142023-12-10T18:54:32.456Z Skipped 0 0 0 0
119152023-12-10T18:54:32.456Z Error 0 0 0 0
119162023-12-10T18:54:32.456ZLast Flush: 1025 1025 0
119172023-12-10T18:54:32.456ZDownstairs last five completed:
119182023-12-10T18:54:32.456ZUpstairs last five completed: 30 29 28 27 26 25
119192023-12-10T18:54:32.456ZWrote out file "/tmp/test_repair_verify.data"
119202023-12-10T18:54:32.456Z
119212023-12-10T18:54:32.456ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
119222023-12-10T18:54:32.456ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
119232023-12-10T18:54:32.456Z 0 000-019 90 90 90 162 162 162 F F F
119242023-12-10T18:54:32.456Z 1 020-039 96 96 90 174 174 162 F F F <---
119252023-12-10T18:54:32.457Z 2 040-059 96 96 94 174 174 171 F F F <---
119262023-12-10T18:54:32.457Z 3 060-079 91 91 91 163 163 163 F F F
119272023-12-10T18:54:32.457Z 4 080-099 95 95 95 172 172 172 F F F
119282023-12-10T18:54:32.458Z 5 100-119 85 85 85 149 149 149 F F F
119292023-12-10T18:54:32.459Z 6 120-139 94 94 94 168 168 168 F F F
119302023-12-10T18:54:32.459Z 7 140-159 92 92 92 165 165 165 F F F
119312023-12-10T18:54:32.459Z 8 160-179 88 88 88 152 152 152 F F F
119322023-12-10T18:54:32.459Z 9 180-199 92 92 92 165 165 165 F F F
119332023-12-10T18:54:32.459Z 10 200-219 96 96 94 174 174 168 F F F <---
119342023-12-10T18:54:32.459Z 11 220-239 96 96 95 173 173 172 T T T <---
119352023-12-10T18:54:32.459Z 12 240-259 94 94 94 170 170 170 F F F
119362023-12-10T18:54:32.460Z 13 260-279 94 94 94 168 168 168 F F F
119372023-12-10T18:54:32.460Z 14 280-299 94 94 94 168 168 168 F F F
119382023-12-10T18:54:32.460Z 15 300-319 91 91 91 163 163 163 F F F
119392023-12-10T18:54:32.460Z 16 320-339 91 91 91 163 163 163 F F F
119402023-12-10T18:54:32.460Z 17 340-359 96 96 95 174 174 172 F F T <---
119412023-12-10T18:54:32.460Z 18 360-379 91 91 91 163 163 163 F F F
119422023-12-10T18:54:32.460Z 19 380-399 90 90 90 158 158 158 T T F <---
119432023-12-10T18:54:32.460Z 20 400-419 90 90 90 161 161 161 F F F
119442023-12-10T18:54:32.460Z 21 420-439 95 95 95 172 172 172 F F F
119452023-12-10T18:54:32.460Z 22 440-459 96 96 95 174 174 172 F F F <---
119462023-12-10T18:54:32.460Z 23 460-479 96 96 82 174 174 145 F F F <---
119472023-12-10T18:54:32.460Z 24 480-499 86 86 86 150 150 150 F F F
119482023-12-10T18:54:32.460Z 25 500-519 92 92 92 165 165 165 F F F
119492023-12-10T18:54:32.460Z 26 520-539 96 96 93 173 173 167 F F F <---
119502023-12-10T18:54:32.465Z 27 540-559 92 92 92 165 165 165 F F F
119512023-12-10T18:54:32.465Z 28 560-579 94 94 94 171 171 171 F F F
119522023-12-10T18:54:32.465Z 29 580-599 96 96 92 174 174 166 F F F <---
119532023-12-10T18:54:32.465ZMax gen: 96, Max flush: 174
119542023-12-10T18:54:32.465ZError: Difference in extent metadata found!
119552023-12-10T18:54:32.465ZA Difference in extent metadata is expected here
119562023-12-10T18:54:32.465ZOn loop 48
119572023-12-10T18:54:32.465Z
119582023-12-10T18:54:32.466ZVerifying data now
119592023-12-10T18:54:37.458ZNo change after verify
119602023-12-10T18:54:37.460ZLoop: 48 Downstairs dump after verify (and repair):
119612023-12-10T18:54:37.482ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
119622023-12-10T18:54:37.482Z 0 000-019 90 90 90 162 162 162 F F F
119632023-12-10T18:54:37.487Z 1 020-039 96 96 96 174 174 174 F F F
119642023-12-10T18:54:37.487Z 2 040-059 96 96 96 174 174 174 F F F
119652023-12-10T18:54:37.487Z 3 060-079 91 91 91 163 163 163 F F F
119662023-12-10T18:54:37.487Z 4 080-099 95 95 95 172 172 172 F F F
119672023-12-10T18:54:37.488Z 5 100-119 85 85 85 149 149 149 F F F
119682023-12-10T18:54:37.488Z 6 120-139 94 94 94 168 168 168 F F F
119692023-12-10T18:54:37.488Z 7 140-159 92 92 92 165 165 165 F F F
119702023-12-10T18:54:37.488Z 8 160-179 88 88 88 152 152 152 F F F
119712023-12-10T18:54:37.488Z 9 180-199 92 92 92 165 165 165 F F F
119722023-12-10T18:54:37.488Z 10 200-219 96 96 96 174 174 174 F F F
119732023-12-10T18:54:37.488Z 11 220-239 97 97 97 175 175 175 F F F
119742023-12-10T18:54:37.488Z 12 240-259 94 94 94 170 170 170 F F F
119752023-12-10T18:54:37.488Z 13 260-279 94 94 94 168 168 168 F F F
119762023-12-10T18:54:37.488Z 14 280-299 94 94 94 168 168 168 F F F
119772023-12-10T18:54:37.489Z 15 300-319 91 91 91 163 163 163 F F F
119782023-12-10T18:54:37.489Z 16 320-339 91 91 91 163 163 163 F F F
119792023-12-10T18:54:37.489Z 17 340-359 96 96 96 174 174 174 F F F
119802023-12-10T18:54:37.489Z 18 360-379 91 91 91 163 163 163 F F F
119812023-12-10T18:54:37.489Z 19 380-399 97 97 97 175 175 175 F F F
119822023-12-10T18:54:37.489Z 20 400-419 90 90 90 161 161 161 F F F
119832023-12-10T18:54:37.489Z 21 420-439 95 95 95 172 172 172 F F F
119842023-12-10T18:54:37.489Z 22 440-459 96 96 96 174 174 174 F F F
119852023-12-10T18:54:37.489Z 23 460-479 96 96 96 174 174 174 F F F
119862023-12-10T18:54:37.490Z 24 480-499 86 86 86 150 150 150 F F F
119872023-12-10T18:54:37.490Z 25 500-519 92 92 92 165 165 165 F F F
119882023-12-10T18:54:37.490Z 26 520-539 96 96 96 173 173 173 F F F
119892023-12-10T18:54:37.490Z 27 540-559 92 92 92 165 165 165 F F F
119902023-12-10T18:54:37.490Z 28 560-579 94 94 94 171 171 171 F F F
119912023-12-10T18:54:37.490Z 29 580-599 96 96 96 174 174 174 F F F
119922023-12-10T18:54:37.490ZMax gen: 97, Max flush: 175
119932023-12-10T18:54:37.494Z
119942023-12-10T18:54:37.494ZBegin loop 49
119952023-12-10T18:54:37.495ZDownstairs to restart: 2
119962023-12-10T18:54:37.515Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.520389631Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554}
119972023-12-10T18:54:37.515Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.520781851Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554}
119982023-12-10T18:54:37.521Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.520833096Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554}
119992023-12-10T18:54:37.522Z{"msg":"Crucible 06ac998d-739a-4845-acf7-e8acbace7165 has session id: 61067017-3902-4f65-86fb-8981724028e9","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.52089839Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120002023-12-10T18:54:37.522ZCrucible runtime is spawned
120012023-12-10T18:54:37.522Z{"msg":"Upstairs opts: Upstairs UUID: 06ac998d-739a-4845-acf7-e8acbace7165, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.520996627Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120022023-12-10T18:54:37.523Z{"msg":"Crucible stats registered with UUID: 06ac998d-739a-4845-acf7-e8acbace7165","v":0,"name":"crucible","level":30The guest has requested activation with gen:98
120032023-12-10T18:54:37.523Z,"time":"2023-12-10T18:54:37.521040389Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120042023-12-10T18:54:37.523Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.521161123Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9","task":"up_listen"}
120052023-12-10T18:54:37.524Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.521232034Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120062023-12-10T18:54:37.524Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.521258776Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"looper":"2","session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120072023-12-10T18:54:37.524Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.521326507Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120082023-12-10T18:54:37.524Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.5213926Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"looper":"0","session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120092023-12-10T18:54:37.524Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.5214993Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"looper":"1","session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120102023-12-10T18:54:37.524Z{"msg":"[0] 06ac998d-739a-4845-acf7-e8acbace7165 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.521551854Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"looper":"0","session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120112023-12-10T18:54:37.524Z{"msg":"[1] 06ac998d-739a-4845-acf7-e8acbace7165 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.521725431Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"looper":"1","session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120122023-12-10T18:54:37.524Z{"msg":"06ac998d-739a-4845-acf7-e8acbace7165 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.521990255Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120132023-12-10T18:54:37.524Z{"msg":"Set desired generation to :98","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.522045183Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120142023-12-10T18:54:37.525Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.522162352Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120152023-12-10T18:54:37.525Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.522472538Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120162023-12-10T18:54:37.525Z{"msg":"[0] 06ac998d-739a-4845-acf7-e8acbace7165 (61067017-3902-4f65-86fb-8981724028e9) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.522839652Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120172023-12-10T18:54:37.525Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.522898656Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120182023-12-10T18:54:37.525Z{"msg":"[0] client is_active_req TRUE, promote! session 61067017-3902-4f65-86fb-8981724028e9","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.522972625Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120192023-12-10T18:54:37.525Z{"msg":"[1] 06ac998d-739a-4845-acf7-e8acbace7165 (61067017-3902-4f65-86fb-8981724028e9) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.523100716Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120202023-12-10T18:54:37.525Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.523156673Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120212023-12-10T18:54:37.525Z{"msg":"[1] client is_active_req TRUE, promote! session 61067017-3902-4f65-86fb-8981724028e9","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.523182274Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120222023-12-10T18:54:37.525Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.523676477Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120232023-12-10T18:54:37.526Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.523769106Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120242023-12-10T18:54:37.527Z{"msg":"06ac998d-739a-4845-acf7-e8acbace7165 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.523805541Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120252023-12-10T18:54:37.527Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.523865572Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120262023-12-10T18:54:37.527Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.523914008Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120272023-12-10T18:54:37.527Z{"msg":"06ac998d-739a-4845-acf7-e8acbace7165 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.523953026Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120282023-12-10T18:54:37.528Z{"msg":"[0] 06ac998d-739a-4845-acf7-e8acbace7165 (61067017-3902-4f65-86fb-8981724028e9) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.524421067Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120292023-12-10T18:54:37.528Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.524456068Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120302023-12-10T18:54:37.529Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:37.524476908Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120312023-12-10T18:54:37.529Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.52449684Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120322023-12-10T18:54:37.530Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.524614824Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120332023-12-10T18:54:37.530Z{"msg":"06ac998d-739a-4845-acf7-e8acbace7165 WaitQuorum WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.52467147Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120342023-12-10T18:54:37.531Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.52469569Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120352023-12-10T18:54:37.531Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.524715446Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120362023-12-10T18:54:37.531Z{"msg":"[1] 06ac998d-739a-4845-acf7-e8acbace7165 (61067017-3902-4f65-86fb-8981724028e9) WaitQuorum WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.524759058Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120372023-12-10T18:54:37.531Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.524782538Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120382023-12-10T18:54:37.532Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:37.524823342Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120392023-12-10T18:54:37.532Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.524845329Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120402023-12-10T18:54:37.532Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.524885954Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120412023-12-10T18:54:37.532Z{"msg":"06ac998d-739a-4845-acf7-e8acbace7165 WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.524909026Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120422023-12-10T18:54:37.533Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.524949591Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120432023-12-10T18:54:37.534Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:37.524971256Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120442023-12-10T18:54:38.518Z{"msg":"[2] 06ac998d-739a-4845-acf7-e8acbace7165 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.522658181Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"looper":"2","session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120452023-12-10T18:54:38.518Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.522743957Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120462023-12-10T18:54:38.523Z{"msg":"[2] 06ac998d-739a-4845-acf7-e8acbace7165 (61067017-3902-4f65-86fb-8981724028e9) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.523796927Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120472023-12-10T18:54:38.523Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.523852906Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120482023-12-10T18:54:38.524Z{"msg":"[2] client is_active_req TRUE, promote! session 61067017-3902-4f65-86fb-8981724028e9","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.523878227Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120492023-12-10T18:54:38.524Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.524507835Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120502023-12-10T18:54:38.524Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.524563986Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120512023-12-10T18:54:38.524Z{"msg":"06ac998d-739a-4845-acf7-e8acbace7165 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.524589479Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120522023-12-10T18:54:38.524Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.525234518Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120532023-12-10T18:54:38.524Z{"msg":"[2] 06ac998d-739a-4845-acf7-e8acbace7165 (61067017-3902-4f65-86fb-8981724028e9) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.525405094Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120542023-12-10T18:54:38.524Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.525460869Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120552023-12-10T18:54:38.524Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:38.525485197Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120562023-12-10T18:54:38.524Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.52553153Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120572023-12-10T18:54:38.524Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.525556988Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120582023-12-10T18:54:38.525Z{"msg":"06ac998d-739a-4845-acf7-e8acbace7165 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.525592925Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120592023-12-10T18:54:38.525Z{"msg":"[0]R flush_numbers[0..12]: [162, 174, 174, 163, 172, 149, 168, 165, 152, 165, 174, 175]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.525615118Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120602023-12-10T18:54:38.525Z{"msg":"[0]R generation[0..12]: [90, 96, 96, 91, 95, 85, 94, 92, 88, 92, 96, 97]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.525657575Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120612023-12-10T18:54:38.525Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.525681754Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120622023-12-10T18:54:38.525Z{"msg":"[1]R flush_numbers[0..12]: [162, 174, 174, 163, 172, 149, 168, 165, 152, 165, 174, 175]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.525723212Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120632023-12-10T18:54:38.525Z{"msg":"[1]R generation[0..12]: [90, 96, 96, 91, 95, 85, 94, 92, 88, 92, 96, 97]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.525746396Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120642023-12-10T18:54:38.525Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.525776609Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120652023-12-10T18:54:38.525Z{"msg":"[2]R flush_numbers[0..12]: [162, 174, 174, 163, 172, 149, 168, 165, 152, 165, 174, 175]","v":0,"The guest has finished waiting for activation with:98
120662023-12-10T18:54:38.525Zname":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
120672023-12-10T18:54:38.525Z,"time":"2023-12-10T18:54:38.525869603Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120682023-12-10T18:54:38.526Z{"msg":"[2]R generation[0..12]: [90, 96, 96, 91, 95, 85, 94, 92, 88, 92, 96, 97]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.525968111Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120692023-12-10T18:54:38.526Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.525997568Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120702023-12-10T18:54:38.526Z{"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.526018052Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120712023-12-10T18:54:38.526Z{"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.526052554Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120722023-12-10T18:54:38.526Z{"msg":"Next flush: 176","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.526087273Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120732023-12-10T18:54:38.528Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.526122855Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120742023-12-10T18:54:38.528Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.526207526Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120752023-12-10T18:54:38.528Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.52624286Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120762023-12-10T18:54:38.528Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.526287787Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120772023-12-10T18:54:38.528Z{"msg":"06ac998d-739a-4845-acf7-e8acbace7165 is now active with session: 61067017-3902-4f65-86fb-8981724028e9","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.526322372Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120782023-12-10T18:54:38.528Z{"msg":"06ac998d-739a-4845-acf7-e8acbace7165 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.526363717Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120792023-12-10T18:54:38.529Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.526403048Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120802023-12-10T18:54:38.529Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.52643667Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120812023-12-10T18:54:38.529Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.526468662Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120822023-12-10T18:54:38.529Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.52649336Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120832023-12-10T18:54:38.529Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.526513406Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120842023-12-10T18:54:38.531Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.526533315Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120852023-12-10T18:54:38.531Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.526560444Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120862023-12-10T18:54:38.532Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.526591848Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120872023-12-10T18:54:38.532Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.526616578Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120882023-12-10T18:54:38.532Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.526636908Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120892023-12-10T18:54:38.532Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:38.526658148Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1554,"session_id":"61067017-3902-4f65-86fb-8981724028e9"}
120902023-12-10T18:54:38.532ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
120912023-12-10T18:54:38.537ZLoading write count information from file "/tmp/test_repair_verify.data"
120922023-12-10T18:54:38.537ZRun Repair workload
120932023-12-10T18:54:38.537Z01/30 Write block 583 len 36864 data: 6 6 4 6 6 7 7 8 7
120942023-12-10T18:54:38.538Z02/30 Read block 140 len 24576
120952023-12-10T18:54:38.538Z03/30 Write block 355 len 20480 data: 8 9 8 8 8
120962023-12-10T18:54:38.538Z04/30 Write block 282 len 8192 data: 7 8
120972023-12-10T18:54:38.538Z05/30 Write block 206 len 24576 data: 7 7 6 5 4 5
120982023-12-10T18:54:38.539Z06/30 Read block 456 len 4096
120992023-12-10T18:54:38.539Z07/30 Read block 555 len 40960
121002023-12-10T18:54:38.539Z08/30 Write block 386 len 28672 data: 10 9 9 8 9 7 8
121012023-12-10T18:54:38.543Z09/30 Write block 299 len 32768 data: 9 7 8 8 10 11 7 8
121022023-12-10T18:54:38.543Z10/30 Flush
121032023-12-10T18:54:38.543Z11/30 Write block 398 len 24576 data: 6 7 7 7 8 9
121042023-12-10T18:54:38.543Z12/30 Read block 459 len 36864
121052023-12-10T18:54:38.547Z13/30 Read block 485 len 16384
121062023-12-10T18:54:38.563Z14/30 Write block 92 len 20480 data: 6 8 8 7 6
121072023-12-10T18:54:38.563Z15/30 Write block 215 len 28672 data: 11 13 13 12 10 10 13
121082023-12-10T18:54:38.563Z16/30 Read block 32 len 4096
121092023-12-10T18:54:38.567Z17/30 Write block 73 len 8192 data: 7 9
121102023-12-10T18:54:38.567Z18/30 Write block 401 len 12288 data: 8 9 10
121112023-12-10T18:54:38.567Z19/30 Read block 65 len 8192
121122023-12-10T18:54:38.567Z20/30 Flush
121132023-12-10T18:54:38.568Z21/30 Write block 201 len 20480 data: 12 10 12 10 10
121142023-12-10T18:54:38.568Z22/30 Write block 345 len 28672 data: 10 9 10 8 11 9 10
121152023-12-10T18:54:38.568Z23/30 Write block 250 len 20480 data: 11 11 11 12 9
121162023-12-10T18:54:38.568Z24/30 Read block 496 len 32768
121172023-12-10T18:54:38.568Z25/30 Read block 10 len 32768
121182023-12-10T18:54:38.568Z26/30 Read block 496 len 20480
121192023-12-10T18:54:38.571Z27/30 Read block 518 len 20480
121202023-12-10T18:54:38.572Z28/30 Read block 430 len 16384
121212023-12-10T18:54:38.576Z29/30 Write block 77 len 40960 data: 9 9 11 10 10 10 10 10 12 12
121222023-12-10T18:54:38.576Z30/30 Write block 584 len 40960 data: 7 5 7 7 8 8 9 8 7 6
121232023-12-10T18:54:38.577Z----------------------------------------------------------------
121242023-12-10T18:54:38.577Z Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30
121252023-12-10T18:54:38.577ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
121262023-12-10T18:54:38.577Z 1 Acked 1000 Write 9 Done Done Done false
121272023-12-10T18:54:38.577Z 2 Acked 1001 Read 6 Done Done Done false
121282023-12-10T18:54:38.577Z 3 Acked 1002 Write 5 Done Done Sent false
121292023-12-10T18:54:38.577Z 4 Acked 1003 Write 2 Done Done Sent false
121302023-12-10T18:54:38.577Z 5 Acked 1004 Write 6 Done Done Sent false
121312023-12-10T18:54:38.577Z 6 Acked 1005 Read 1 Done Done Sent false
121322023-12-10T18:54:38.578Z 7 Acked 1006 Read 10 Done Done Sent false
121332023-12-10T18:54:38.578Z 8 Acked 1007 Write 7 Done Done Sent false
121342023-12-10T18:54:38.578Z 9 Acked 1008 Write 8 Done Done Sent false
121352023-12-10T18:54:38.578Z 10 Acked 1009 Flush 0 Done Done Sent false
121362023-12-10T18:54:38.578Z 11 Acked 1010 Write 6 Done Done Sent false
121372023-12-10T18:54:38.578Z 12 Acked 1011 Read 9 Done Done Sent false
121382023-12-10T18:54:38.578Z 13 Acked 1012 Read 4 Done Done Sent false
121392023-12-10T18:54:38.578Z 14 Acked 1013 Write 5 Done Done Sent false
121402023-12-10T18:54:38.578Z 15 Acked 1014 Write 7 Done Done Sent false
121412023-12-10T18:54:38.578Z 16 Acked 1015 Read 1 Done Done Sent false
121422023-12-10T18:54:38.579Z 17 Acked 1016 Write 2 Done Done Sent false
121432023-12-10T18:54:38.579Z 18 Acked 1017 Write 3 Done Done Sent false
121442023-12-10T18:54:38.579Z 19 Acked 1018 Read 2 Done Done Sent false
121452023-12-10T18:54:38.579Z 20 Acked 1019 Flush 0 Done Done Sent false
121462023-12-10T18:54:38.579Z 21 Acked 1020 Write 5 Done Done Sent false
121472023-12-10T18:54:38.579Z 22 Acked 1021 Write 7 Done Done Sent false
121482023-12-10T18:54:38.580Z 23 Acked 1022 Write 5 Done Done Sent false
121492023-12-10T18:54:38.580Z 24 Acked 1023 Read 8 Done Done Sent false
121502023-12-10T18:54:38.580Z 25 Acked 1024 Read 8 Done Done Sent false
121512023-12-10T18:54:38.580Z 26 Acked 1025 Read 5 Done Done Sent false
121522023-12-10T18:54:38.580Z 27 Acked 1026 Read 5 Done Done Sent false
121532023-12-10T18:54:38.580Z 28 Acked 1027 Read 4 Done Done Sent false
121542023-12-10T18:54:38.580Z 29 Acked 1028 Write 10 Sent Done Sent false
121552023-12-10T18:54:38.580Z 30 Acked 1029 Write 10 Sent Sent Sent false
121562023-12-10T18:54:38.581Z STATES DS:0 DS:1 DS:2 TOTAL
121572023-12-10T18:54:38.581Z New 0 0 0 0
121582023-12-10T18:54:38.581Z Sent 2 1 28 31
121592023-12-10T18:54:38.581Z Done 28 29 2 59
121602023-12-10T18:54:38.581Z Skipped 0 0 0 0
121612023-12-10T18:54:38.581Z Error 0 0 0 0
121622023-12-10T18:54:38.581ZLast Flush: 1019 1019 0
121632023-12-10T18:54:38.581ZDownstairs last five completed:
121642023-12-10T18:54:38.582ZUpstairs last five completed: 30 29 28 27 26 25
121652023-12-10T18:54:38.582ZWrote out file "/tmp/test_repair_verify.data"
121662023-12-10T18:54:38.586Z
121672023-12-10T18:54:38.589ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
121682023-12-10T18:54:38.615ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
121692023-12-10T18:54:38.615Z 0 000-019 90 90 90 162 162 162 F F F
121702023-12-10T18:54:38.615Z 1 020-039 96 96 96 174 174 174 F F F
121712023-12-10T18:54:38.615Z 2 040-059 96 96 96 174 174 174 F F F
121722023-12-10T18:54:38.616Z 3 060-079 98 98 91 177 177 163 T T F <---
121732023-12-10T18:54:38.617Z 4 080-099 98 98 95 177 177 172 T T F <---
121742023-12-10T18:54:38.621Z 5 100-119 85 85 85 149 149 149 F F F
121752023-12-10T18:54:38.621Z 6 120-139 94 94 94 168 168 168 F F F
121762023-12-10T18:54:38.621Z 7 140-159 92 92 92 165 165 165 F F F
121772023-12-10T18:54:38.621Z 8 160-179 88 88 88 152 152 152 F F F
121782023-12-10T18:54:38.621Z 9 180-199 92 92 92 165 165 165 F F F
121792023-12-10T18:54:38.621Z 10 200-219 98 98 96 177 177 174 T T F <---
121802023-12-10T18:54:38.622Z 11 220-239 98 98 97 177 177 175 F F F <---
121812023-12-10T18:54:38.622Z 12 240-259 94 94 94 170 170 170 T T F <---
121822023-12-10T18:54:38.622Z 13 260-279 94 94 94 168 168 168 F F F
121832023-12-10T18:54:38.622Z 14 280-299 98 98 94 176 176 168 F F F <---
121842023-12-10T18:54:38.622Z 15 300-319 98 98 91 176 176 163 F F F <---
121852023-12-10T18:54:38.622Z 16 320-339 91 91 91 163 163 163 F F F
121862023-12-10T18:54:38.622Z 17 340-359 98 98 96 176 176 174 T T F <---
121872023-12-10T18:54:38.623Z 18 360-379 91 91 91 163 163 163 F F F
121882023-12-10T18:54:38.623Z 19 380-399 98 98 97 177 177 175 F F F <---
121892023-12-10T18:54:38.623Z 20 400-419 98 98 90 177 177 161 F F F <---
121902023-12-10T18:54:38.623Z 21 420-439 95 95 95 172 172 172 F F F
121912023-12-10T18:54:38.623Z 22 440-459 96 96 96 174 174 174 F F F
121922023-12-10T18:54:38.623Z 23 460-479 96 96 96 174 174 174 F F F
121932023-12-10T18:54:38.623Z 24 480-499 86 86 86 150 150 150 F F F
121942023-12-10T18:54:38.623Z 25 500-519 92 92 92 165 165 165 F F F
121952023-12-10T18:54:38.623Z 26 520-539 96 96 96 173 173 173 F F F
121962023-12-10T18:54:38.624Z 27 540-559 92 92 92 165 165 165 F F F
121972023-12-10T18:54:38.625Z 28 560-579 94 94 94 171 171 171 F F F
121982023-12-10T18:54:38.625Z 29 580-599 98 98 96 176 176 174 T T T <---
121992023-12-10T18:54:38.625ZMax gen: 98, Max flush: 177
122002023-12-10T18:54:38.625ZError: Difference in extent metadata found!
122012023-12-10T18:54:38.625ZA Difference in extent metadata is expected here
122022023-12-10T18:54:38.626ZOn loop 49
122032023-12-10T18:54:38.626Z
122042023-12-10T18:54:38.630ZVerifying data now
122052023-12-10T18:54:43.730ZNo change after verify
122062023-12-10T18:54:43.730ZLoop: 49 Downstairs dump after verify (and repair):
122072023-12-10T18:54:43.753ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
122082023-12-10T18:54:43.753Z 0 000-019 90 90 90 162 162 162 F F F
122092023-12-10T18:54:43.756Z 1 020-039 96 96 96 174 174 174 F F F
122102023-12-10T18:54:43.756Z 2 040-059 96 96 96 174 174 174 F F F
122112023-12-10T18:54:43.756Z 3 060-079 99 99 99 178 178 178 F F F
122122023-12-10T18:54:43.756Z 4 080-099 99 99 99 178 178 178 F F F
122132023-12-10T18:54:43.757Z 5 100-119 85 85 85 149 149 149 F F F
122142023-12-10T18:54:43.757Z 6 120-139 94 94 94 168 168 168 F F F
122152023-12-10T18:54:43.757Z 7 140-159 92 92 92 165 165 165 F F F
122162023-12-10T18:54:43.757Z 8 160-179 88 88 88 152 152 152 F F F
122172023-12-10T18:54:43.757Z 9 180-199 92 92 92 165 165 165 F F F
122182023-12-10T18:54:43.757Z 10 200-219 99 99 99 178 178 178 F F F
122192023-12-10T18:54:43.757Z 11 220-239 98 98 98 177 177 177 F F F
122202023-12-10T18:54:43.757Z 12 240-259 99 99 99 178 178 178 F F F
122212023-12-10T18:54:43.757Z 13 260-279 94 94 94 168 168 168 F F F
122222023-12-10T18:54:43.757Z 14 280-299 98 98 98 176 176 176 F F F
122232023-12-10T18:54:43.757Z 15 300-319 98 98 98 176 176 176 F F F
122242023-12-10T18:54:43.757Z 16 320-339 91 91 91 163 163 163 F F F
122252023-12-10T18:54:43.757Z 17 340-359 99 99 99 178 178 178 F F F
122262023-12-10T18:54:43.758Z 18 360-379 91 91 91 163 163 163 F F F
122272023-12-10T18:54:43.758Z 19 380-399 98 98 98 177 177 177 F F F
122282023-12-10T18:54:43.758Z 20 400-419 98 98 98 177 177 177 F F F
122292023-12-10T18:54:43.758Z 21 420-439 95 95 95 172 172 172 F F F
122302023-12-10T18:54:43.758Z 22 440-459 96 96 96 174 174 174 F F F
122312023-12-10T18:54:43.758Z 23 460-479 96 96 96 174 174 174 F F F
122322023-12-10T18:54:43.758Z 24 480-499 86 86 86 150 150 150 F F F
122332023-12-10T18:54:43.758Z 25 500-519 92 92 92 165 165 165 F F F
122342023-12-10T18:54:43.758Z 26 520-539 96 96 96 173 173 173 F F F
122352023-12-10T18:54:43.759Z 27 540-559 92 92 92 165 165 165 F F F
122362023-12-10T18:54:43.759Z 28 560-579 94 94 94 171 171 171 F F F
122372023-12-10T18:54:43.759Z 29 580-599 99 99 99 178 178 178 F F F
122382023-12-10T18:54:43.759ZMax gen: 99, Max flush: 178
122392023-12-10T18:54:43.762Z
122402023-12-10T18:54:43.762ZBegin loop 50
122412023-12-10T18:54:43.762ZDownstairs to restart: 1
122422023-12-10T18:54:43.790Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.795268664Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562}
122432023-12-10T18:54:43.790Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.795694863Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562}
122442023-12-10T18:54:43.790Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.795734958Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562}
122452023-12-10T18:54:43.793Z{"msg":"Crucible 2a1b0077-65c0-42e9-aa8c-b587881edd9a has session id: c4974982-b2e1-42e8-9933-ee8ebf049090","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.79578557Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122462023-12-10T18:54:43.793ZCrucible runtime is spawned
122472023-12-10T18:54:43.793Z{"msg":"Upstairs opts: Upstairs UUID: 2a1b0077-65c0-42e9-aa8c-b587881edd9a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.796030919Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122482023-12-10T18:54:43.793Z{The guest has requested activation with gen:100
122492023-12-10T18:54:43.793Z"msg":"Crucible stats registered with UUID: 2a1b0077-65c0-42e9-aa8c-b587881edd9a","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.796177039Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122502023-12-10T18:54:43.793Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.796260848Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090","task":"up_listen"}
122512023-12-10T18:54:43.793Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.796304724Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122522023-12-10T18:54:43.793Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.796380277Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122532023-12-10T18:54:43.793Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.796419596Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"looper":"0","session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122542023-12-10T18:54:43.794Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.796508663Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"looper":"2","session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122552023-12-10T18:54:43.794Z{"msg":"[0] 2a1b0077-65c0-42e9-aa8c-b587881edd9a looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.796566964Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"looper":"0","session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122562023-12-10T18:54:43.794Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.796647602Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"looper":"1","session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122572023-12-10T18:54:43.795Z{"msg":"[2] 2a1b0077-65c0-42e9-aa8c-b587881edd9a looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.796690602Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"looper":"2","session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122582023-12-10T18:54:43.795Z{"msg":"[1] 2a1b0077-65c0-42e9-aa8c-b587881edd9a looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.796845991Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"looper":"1","session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122592023-12-10T18:54:43.795Z{"msg":"2a1b0077-65c0-42e9-aa8c-b587881edd9a active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.797167368Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122602023-12-10T18:54:43.795Z{"msg":"Set desired generation to :100","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.797220886Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122612023-12-10T18:54:43.795Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.797252367Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122622023-12-10T18:54:43.795Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.797602969Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122632023-12-10T18:54:43.795Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.797702959Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122642023-12-10T18:54:43.795Z{"msg":"[0] 2a1b0077-65c0-42e9-aa8c-b587881edd9a (c4974982-b2e1-42e8-9933-ee8ebf049090) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.798124602Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122652023-12-10T18:54:43.795Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.79820172Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122662023-12-10T18:54:43.795Z{"msg":"[0] client is_active_req TRUE, promote! session c4974982-b2e1-42e8-9933-ee8ebf049090","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.798230035Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122672023-12-10T18:54:43.796Z{"msg":"[2] 2a1b0077-65c0-42e9-aa8c-b587881edd9a (c4974982-b2e1-42e8-9933-ee8ebf049090) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.798269804Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122682023-12-10T18:54:43.796Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.798306573Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122692023-12-10T18:54:43.796Z{"msg":"[2] client is_active_req TRUE, promote! session c4974982-b2e1-42e8-9933-ee8ebf049090","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.798423599Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122702023-12-10T18:54:43.801Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.798945325Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122712023-12-10T18:54:43.801Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.799016725Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122722023-12-10T18:54:43.801Z{"msg":"2a1b0077-65c0-42e9-aa8c-b587881edd9a WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.799092679Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122732023-12-10T18:54:43.801Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.79915217Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122742023-12-10T18:54:43.801Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.799225498Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122752023-12-10T18:54:43.801Z{"msg":"2a1b0077-65c0-42e9-aa8c-b587881edd9a WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.799266904Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122762023-12-10T18:54:43.801Z{"msg":"[1] 2a1b0077-65c0-42e9-aa8c-b587881edd9a (c4974982-b2e1-42e8-9933-ee8ebf049090) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.799308129Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122772023-12-10T18:54:43.802Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.799377321Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122782023-12-10T18:54:43.802Z{"msg":"[1] client is_active_req TRUE, promote! session c4974982-b2e1-42e8-9933-ee8ebf049090","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.799416952Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122792023-12-10T18:54:43.802Z{"msg":"[0] 2a1b0077-65c0-42e9-aa8c-b587881edd9a (c4974982-b2e1-42e8-9933-ee8ebf049090) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.7997186Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122802023-12-10T18:54:43.802Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.799788682Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122812023-12-10T18:54:43.802Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:43.799831745Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122822023-12-10T18:54:43.802Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.799890241Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122832023-12-10T18:54:43.802Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.799931636Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122842023-12-10T18:54:43.802Z{"msg":"2a1b0077-65c0-42e9-aa8c-b587881edd9a WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.799998894Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122852023-12-10T18:54:43.802Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.800038809Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122862023-12-10T18:54:43.802Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.800126482Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122872023-12-10T18:54:43.802Z{"msg":"[2] 2a1b0077-65c0-42e9-aa8c-b587881edd9a (c4974982-b2e1-42e8-9933-ee8ebf049090) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.800173197Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122882023-12-10T18:54:43.802Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.800245274Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122892023-12-10T18:54:43.802Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:43.80028586Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122902023-12-10T18:54:43.802Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.800342118Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122912023-12-10T18:54:43.802Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.800382729Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122922023-12-10T18:54:43.802Z{"msg":"2a1b0077-65c0-42e9-aa8c-b587881edd9a WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.800439933Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122932023-12-10T18:54:43.802Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.800479691Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122942023-12-10T18:54:43.803Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.800534884Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122952023-12-10T18:54:43.803Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.800576759Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122962023-12-10T18:54:43.803Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.800639699Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122972023-12-10T18:54:43.803Z{"msg":"2a1b0077-65c0-42e9-aa8c-b587881edd9a WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.800682188Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122982023-12-10T18:54:43.803Z{"msg":"[1] 2a1b0077-65c0-42e9-aa8c-b587881edd9a (c4974982-b2e1-42e8-9933-ee8ebf049090) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.801017303Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
122992023-12-10T18:54:43.803Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.801082705Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123002023-12-10T18:54:43.803Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:43.801123248Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123012023-12-10T18:54:43.803Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.801179842Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123022023-12-10T18:54:43.803Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.801220137Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123032023-12-10T18:54:43.803Z{"msg":"2a1b0077-65c0-42e9-aa8c-b587881edd9a WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.801276819Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123042023-12-10T18:54:43.803Z{"msg":"[0]R flush_numbers[0..12]: [162, 174, 174, 178, 178, 149, 168, 165, 152, 165, 178, 177]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.801317769Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123052023-12-10T18:54:43.803Z{"msg":"[0]R generation[0..12]: [90, 96, 96, 99, 99, 85, 94, 92, 88, 92, 99, 98]","v":The guest has finished waiting for activation with:100
123062023-12-10T18:54:43.803Z0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
123072023-12-10T18:54:43.803Z,"time":"2023-12-10T18:54:43.801406534Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123082023-12-10T18:54:43.803Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.801451189Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123092023-12-10T18:54:43.803Z{"msg":"[1]R flush_numbers[0..12]: [162, 174, 174, 178, 178, 149, 168, 165, 152, 165, 178, 177]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.801521932Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123102023-12-10T18:54:43.804Z{"msg":"[1]R generation[0..12]: [90, 96, 96, 99, 99, 85, 94, 92, 88, 92, 99, 98]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.801563709Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123112023-12-10T18:54:43.804Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.801601282Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123122023-12-10T18:54:43.804Z{"msg":"[2]R flush_numbers[0..12]: [162, 174, 174, 178, 178, 149, 168, 165, 152, 165, 178, 177]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.801668351Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123132023-12-10T18:54:43.804Z{"msg":"[2]R generation[0..12]: [90, 96, 96, 99, 99, 85, 94, 92, 88, 92, 99, 98]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.801709445Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123142023-12-10T18:54:43.804Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.801768455Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123152023-12-10T18:54:43.804Z{"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.801807709Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123162023-12-10T18:54:43.804Z{"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.80187304Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123172023-12-10T18:54:43.804Z{"msg":"Next flush: 179","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.801912184Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123182023-12-10T18:54:43.805Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.801979688Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123192023-12-10T18:54:43.805Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.802019286Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123202023-12-10T18:54:43.805Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.802083107Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123212023-12-10T18:54:43.805Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.802122328Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123222023-12-10T18:54:43.805Z{"msg":"2a1b0077-65c0-42e9-aa8c-b587881edd9a is now active with session: c4974982-b2e1-42e8-9933-ee8ebf049090","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.802183951Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123232023-12-10T18:54:43.805Z{"msg":"2a1b0077-65c0-42e9-aa8c-b587881edd9a Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.802225355Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123242023-12-10T18:54:43.805Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.802280228Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123252023-12-10T18:54:43.805Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.80231926Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123262023-12-10T18:54:43.806Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.802379828Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123272023-12-10T18:54:43.806Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.802419165Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123282023-12-10T18:54:43.806Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.80247625Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123292023-12-10T18:54:43.806Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
123302023-12-10T18:54:43.806Z,"time":"2023-12-10T18:54:43.802515372Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123312023-12-10T18:54:43.806Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.80266642Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123322023-12-10T18:54:43.806Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.802709623Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123332023-12-10T18:54:43.806Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.802761973Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123342023-12-10T18:54:43.806Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.802802015Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123352023-12-10T18:54:43.807Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:43.80285819Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1562,"session_id":"c4974982-b2e1-42e8-9933-ee8ebf049090"}
123362023-12-10T18:54:43.807ZLoading write count information from file "/tmp/test_repair_verify.data"
123372023-12-10T18:54:43.807ZRun Repair workload
123382023-12-10T18:54:43.807Z01/30 Write block 1 len 12288 data: 3 4 3
123392023-12-10T18:54:43.807Z02/30 Read block 96 len 24576
123402023-12-10T18:54:43.838Z03/30 Write block 20 len 12288 data: 9 7 7
123412023-12-10T18:54:43.838Z04/30 Read block 460 len 28672
123422023-12-10T18:54:43.838Z05/30 Write block 391 len 24576 data: 8 9 6 5 5 5
123432023-12-10T18:54:43.838Z06/30 Read block 238 len 40960
123442023-12-10T18:54:43.838Z07/30 Flush
123452023-12-10T18:54:43.841Z08/30 Write block 460 len 24576 data: 5 4 6 6 6 9
123462023-12-10T18:54:43.842Z09/30 Write block 498 len 4096 data: 6
123472023-12-10T18:54:43.842Z10/30 Read block 276 len 8192
123482023-12-10T18:54:43.842Z11/30 Write block 81 len 16384 data: 11 11 11 11
123492023-12-10T18:54:43.843Z12/30 Read block 318 len 12288
123502023-12-10T18:54:43.843Z13/30 Read block 546 len 32768
123512023-12-10T18:54:43.843Z14/30 Read block 167 len 24576
123522023-12-10T18:54:43.843Z15/30 Write block 384 len 28672 data: 11 11 11 10 10 9 10
123532023-12-10T18:54:43.843Z16/30 Write block 543 len 20480 data: 12 11 7 8 8
123542023-12-10T18:54:43.843Z17/30 Read block 3 len 40960
123552023-12-10T18:54:43.843Z18/30 Write block 356 len 36864 data: 10 9 9 9 7 6 6 6 5
123562023-12-10T18:54:43.843Z19/30 Write block 368 len 36864 data: 5 6 5 6 7 8 9 12 12
123572023-12-10T18:54:43.843Z20/30 Read block 591 len 4096
123582023-12-10T18:54:43.843Z21/30 Write block 275 len 16384 data: 12 11 11 12
123592023-12-10T18:54:43.843Z22/30 Write block 266 len 16384 data: 9 7 8 9
123602023-12-10T18:54:43.843Z23/30 Write block 377 len 24576 data: 11 15 13 14 10 11
123612023-12-10T18:54:43.843Z24/30 Read block 135 len 16384
123622023-12-10T18:54:43.843Z25/30 Write block 171 len 32768 data: 9 9 8 8 6 6 7 5
123632023-12-10T18:54:43.843Z26/30 Write block 200 len 32768 data: 11 13 11 13 11 11 8 8
123642023-12-10T18:54:43.843Z27/30 Write block 173 len 4096 data: 9
123652023-12-10T18:54:43.843Z28/30 Read block 272 len 20480
123662023-12-10T18:54:43.849Z29/30 Read block 367 len 28672
123672023-12-10T18:54:43.849Z30/30 Read block 348 len 36864
123682023-12-10T18:54:43.853Z----------------------------------------------------------------
123692023-12-10T18:54:43.853Z Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30
123702023-12-10T18:54:43.853ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
123712023-12-10T18:54:43.853Z 1 Acked 1000 Write 3 Done Done Done false
123722023-12-10T18:54:43.853Z 2 Acked 1001 Read 6 Done Done Done false
123732023-12-10T18:54:43.853Z 3 Acked 1002 Write 3 Done Done Done false
123742023-12-10T18:54:43.853Z 4 Acked 1003 Read 7 Done Sent Done false
123752023-12-10T18:54:43.853Z 5 Acked 1004 Write 6 Done Sent Done false
123762023-12-10T18:54:43.853Z 6 Acked 1005 Read 10 Done Sent Done false
123772023-12-10T18:54:43.853Z 7 Acked 1006 Flush 0 Done Sent Done false
123782023-12-10T18:54:43.853Z 8 Acked 1007 Write 6 Done Sent Done false
123792023-12-10T18:54:43.853Z 9 Acked 1008 Write 1 Done Sent Done false
123802023-12-10T18:54:43.853Z 10 Acked 1009 Read 2 Done Sent Done false
123812023-12-10T18:54:43.853Z 11 Acked 1010 Write 4 Done Sent Done false
123822023-12-10T18:54:43.853Z 12 Acked 1011 Read 3 Done Sent Done false
123832023-12-10T18:54:43.853Z 13 Acked 1012 Read 8 Done Sent Done false
123842023-12-10T18:54:43.854Z 14 Acked 1013 Read 6 Done Sent Done false
123852023-12-10T18:54:43.854Z 15 Acked 1014 Write 7 Done Sent Done false
123862023-12-10T18:54:43.854Z 16 Acked 1015 Write 5 Done Sent Done false
123872023-12-10T18:54:43.854Z 17 Acked 1016 Read 10 Done Sent Done false
123882023-12-10T18:54:43.854Z 18 Acked 1017 Write 9 Done Sent Done false
123892023-12-10T18:54:43.854Z 19 Acked 1018 Write 9 Done Sent Done false
123902023-12-10T18:54:43.854Z 20 Acked 1019 Read 1 Done Sent Done false
123912023-12-10T18:54:43.854Z 21 Acked 1020 Write 4 Done Sent Done false
123922023-12-10T18:54:43.854Z 22 Acked 1021 Write 4 Done Sent Done false
123932023-12-10T18:54:43.854Z 23 Acked 1022 Write 6 Done Sent Done false
123942023-12-10T18:54:43.855Z 24 Acked 1023 Read 4 Done Sent Done false
123952023-12-10T18:54:43.855Z 25 Acked 1024 Write 8 Done Sent Done false
123962023-12-10T18:54:43.855Z 26 Acked 1025 Write 8 Done Sent Done false
123972023-12-10T18:54:43.855Z 27 Acked 1026 Write 1 Done Sent Done false
123982023-12-10T18:54:43.855Z 28 Acked 1027 Read 5 Done Sent Done false
123992023-12-10T18:54:43.855Z 29 Acked 1028 Read 7 Done Sent Done false
124002023-12-10T18:54:43.855Z 30 Acked 1029 Read 9 Sent Sent Done false
124012023-12-10T18:54:43.855Z STATES DS:0 DS:1 DS:2 TOTAL
124022023-12-10T18:54:43.855Z New 0 0 0 0
124032023-12-10T18:54:43.855Z Sent 1 27 0 28
124042023-12-10T18:54:43.855Z Done 29 3 30 62
124052023-12-10T18:54:43.855Z Skipped 0 0 0 0
124062023-12-10T18:54:43.855Z Error 0 0 0 0
124072023-12-10T18:54:43.856ZLast Flush: 1006 0 1006
124082023-12-10T18:54:43.856ZDownstairs last five completed:
124092023-12-10T18:54:43.856ZUpstairs last five completed: 30 29 28 27 26 25
124102023-12-10T18:54:43.856ZWrote out file "/tmp/test_repair_verify.data"
124112023-12-10T18:54:43.859Z
124122023-12-10T18:54:43.861ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
124132023-12-10T18:54:43.889ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
124142023-12-10T18:54:43.889Z 0 000-019 100 90 100 179 162 179 F T F <---
124152023-12-10T18:54:43.892Z 1 020-039 100 96 100 179 174 179 F T F <---
124162023-12-10T18:54:43.893Z 2 040-059 96 96 96 174 174 174 F F F
124172023-12-10T18:54:43.893Z 3 060-079 99 99 99 178 178 178 F F F
124182023-12-10T18:54:43.893Z 4 080-099 99 99 99 178 178 178 T F T <---
124192023-12-10T18:54:43.894Z 5 100-119 85 85 85 149 149 149 F F F
124202023-12-10T18:54:43.894Z 6 120-139 94 94 94 168 168 168 F F F
124212023-12-10T18:54:43.894Z 7 140-159 92 92 92 165 165 165 F F F
124222023-12-10T18:54:43.894Z 8 160-179 88 88 88 152 152 152 T F T <---
124232023-12-10T18:54:43.894Z 9 180-199 92 92 92 165 165 165 F F F
124242023-12-10T18:54:43.894Z 10 200-219 99 99 99 178 178 178 T F T <---
124252023-12-10T18:54:43.894Z 11 220-239 98 98 98 177 177 177 F F F
124262023-12-10T18:54:43.895Z 12 240-259 99 99 99 178 178 178 F F F
124272023-12-10T18:54:43.895Z 13 260-279 94 94 94 168 168 168 T F T <---
124282023-12-10T18:54:43.895Z 14 280-299 98 98 98 176 176 176 F F F
124292023-12-10T18:54:43.895Z 15 300-319 98 98 98 176 176 176 F F F
124302023-12-10T18:54:43.895Z 16 320-339 91 91 91 163 163 163 F F F
124312023-12-10T18:54:43.895Z 17 340-359 99 99 99 178 178 178 T F T <---
124322023-12-10T18:54:43.895Z 18 360-379 91 91 91 163 163 163 T F T <---
124332023-12-10T18:54:43.895Z 19 380-399 100 98 100 179 177 179 T F T <---
124342023-12-10T18:54:43.895Z 20 400-419 98 98 98 177 177 177 F F F
124352023-12-10T18:54:43.895Z 21 420-439 95 95 95 172 172 172 F F F
124362023-12-10T18:54:43.895Z 22 440-459 96 96 96 174 174 174 F F F
124372023-12-10T18:54:43.895Z 23 460-479 96 96 96 174 174 174 T F T <---
124382023-12-10T18:54:43.895Z 24 480-499 86 86 86 150 150 150 T F T <---
124392023-12-10T18:54:43.895Z 25 500-519 92 92 92 165 165 165 F F F
124402023-12-10T18:54:43.895Z 26 520-539 96 96 96 173 173 173 F F F
124412023-12-10T18:54:43.895Z 27 540-559 92 92 92 165 165 165 T F T <---
124422023-12-10T18:54:43.896Z 28 560-579 94 94 94 171 171 171 F F F
124432023-12-10T18:54:43.896Z 29 580-599 99 99 99 178 178 178 F F F
124442023-12-10T18:54:43.896ZMax gen: 100, Max flush: 179
124452023-12-10T18:54:43.896ZError: Difference in extent metadata found!
124462023-12-10T18:54:43.899ZA Difference in extent metadata is expected here
124472023-12-10T18:54:43.899ZOn loop 50
124482023-12-10T18:54:43.899Z
124492023-12-10T18:54:43.899ZVerifying data now
124502023-12-10T18:54:49.106ZNo change after verify
124512023-12-10T18:54:49.106ZLoop: 50 Downstairs dump after verify (and repair):
124522023-12-10T18:54:49.129ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
124532023-12-10T18:54:49.129Z 0 000-019 100 100 100 179 179 179 F F F
124542023-12-10T18:54:49.133Z 1 020-039 100 100 100 179 179 179 F F F
124552023-12-10T18:54:49.133Z 2 040-059 96 96 96 174 174 174 F F F
124562023-12-10T18:54:49.133Z 3 060-079 99 99 99 178 178 178 F F F
124572023-12-10T18:54:49.133Z 4 080-099 101 101 101 180 180 180 F F F
124582023-12-10T18:54:49.133Z 5 100-119 85 85 85 149 149 149 F F F
124592023-12-10T18:54:49.133Z 6 120-139 94 94 94 168 168 168 F F F
124602023-12-10T18:54:49.133Z 7 140-159 92 92 92 165 165 165 F F F
124612023-12-10T18:54:49.133Z 8 160-179 101 101 101 180 180 180 F F F
124622023-12-10T18:54:49.133Z 9 180-199 92 92 92 165 165 165 F F F
124632023-12-10T18:54:49.133Z 10 200-219 101 101 101 180 180 180 F F F
124642023-12-10T18:54:49.134Z 11 220-239 98 98 98 177 177 177 F F F
124652023-12-10T18:54:49.134Z 12 240-259 99 99 99 178 178 178 F F F
124662023-12-10T18:54:49.134Z 13 260-279 101 101 101 180 180 180 F F F
124672023-12-10T18:54:49.134Z 14 280-299 98 98 98 176 176 176 F F F
124682023-12-10T18:54:49.134Z 15 300-319 98 98 98 176 176 176 F F F
124692023-12-10T18:54:49.134Z 16 320-339 91 91 91 163 163 163 F F F
124702023-12-10T18:54:49.134Z 17 340-359 101 101 101 180 180 180 F F F
124712023-12-10T18:54:49.134Z 18 360-379 101 101 101 180 180 180 F F F
124722023-12-10T18:54:49.134Z 19 380-399 101 101 101 180 180 180 F F F
124732023-12-10T18:54:49.134Z 20 400-419 98 98 98 177 177 177 F F F
124742023-12-10T18:54:49.135Z 21 420-439 95 95 95 172 172 172 F F F
124752023-12-10T18:54:49.135Z 22 440-459 96 96 96 174 174 174 F F F
124762023-12-10T18:54:49.135Z 23 460-479 101 101 101 180 180 180 F F F
124772023-12-10T18:54:49.135Z 24 480-499 101 101 101 180 180 180 F F F
124782023-12-10T18:54:49.135Z 25 500-519 92 92 92 165 165 165 F F F
124792023-12-10T18:54:49.135Z 26 520-539 96 96 96 173 173 173 F F F
124802023-12-10T18:54:49.135Z 27 540-559 101 101 101 180 180 180 F F F
124812023-12-10T18:54:49.135Z 28 560-579 94 94 94 171 171 171 F F F
124822023-12-10T18:54:49.135Z 29 580-599 99 99 99 178 178 178 F F F
124832023-12-10T18:54:49.136ZMax gen: 101, Max flush: 180
124842023-12-10T18:54:49.139Z
124852023-12-10T18:54:49.139ZBegin loop 51
124862023-12-10T18:54:49.139ZDownstairs to restart: 2
124872023-12-10T18:54:49.163Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.169095972Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570}
124882023-12-10T18:54:49.163Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.169472418Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570}
124892023-12-10T18:54:49.168Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.169499626Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570}
124902023-12-10T18:54:49.168Z{"msg":"Crucible 28801844-f108-4494-b663-0e921f8fd9ee has session id: 075da191-b9d1-4f2d-b196-81c5a78085ca","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.169542633Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
124912023-12-10T18:54:49.168Z{"msg":"Upstairs opts: Upstairs UUID: 28801844-f108-4494-b663-0e921f8fd9ee, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.169580283Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
124922023-12-10T18:54:49.168Z{"msg":"Crucible stats registered with UUID: 28801844-f108-4494-b663-0e921f8fd9ee","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.169620209Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
124932023-12-10T18:54:49.168ZCrucible runtime is spawned
124942023-12-10T18:54:49.168ZThe guest has requested activation with gen:102
124952023-12-10T18:54:49.168Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.170025688Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca","task":"up_listen"}
124962023-12-10T18:54:49.169Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.170085374Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"looper":"1","session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
124972023-12-10T18:54:49.169Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.170118188Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
124982023-12-10T18:54:49.169Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.1701984Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
124992023-12-10T18:54:49.169Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.170389926Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"looper":"2","session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125002023-12-10T18:54:49.169Z{"msg":"[1] 28801844-f108-4494-b663-0e921f8fd9ee looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.170486562Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"looper":"1","session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125012023-12-10T18:54:49.169Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.17057727Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"looper":"0","session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125022023-12-10T18:54:49.169Z{"msg":"[2] 28801844-f108-4494-b663-0e921f8fd9ee looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.170604999Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"looper":"2","session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125032023-12-10T18:54:49.169Z{"msg":"[0] 28801844-f108-4494-b663-0e921f8fd9ee looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.17087844Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"looper":"0","session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125042023-12-10T18:54:49.169Z{"msg":"28801844-f108-4494-b663-0e921f8fd9ee active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.171405055Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125052023-12-10T18:54:49.170Z{"msg":"Set desired generation to :102","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.171502686Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125062023-12-10T18:54:49.170Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.171546863Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125072023-12-10T18:54:49.170Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.171925788Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125082023-12-10T18:54:49.170Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.172007662Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125092023-12-10T18:54:49.170Z{"msg":"[1] 28801844-f108-4494-b663-0e921f8fd9ee (075da191-b9d1-4f2d-b196-81c5a78085ca) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.172189663Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125102023-12-10T18:54:49.170Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.172308348Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125112023-12-10T18:54:49.170Z{"msg":"[1] client is_active_req TRUE, promote! session 075da191-b9d1-4f2d-b196-81c5a78085ca","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.172402067Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125122023-12-10T18:54:49.170Z{"msg":"[0] 28801844-f108-4494-b663-0e921f8fd9ee (075da191-b9d1-4f2d-b196-81c5a78085ca) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.172481634Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125132023-12-10T18:54:49.170Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.172556366Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125142023-12-10T18:54:49.171Z{"msg":"[0] client is_active_req TRUE, promote! session 075da191-b9d1-4f2d-b196-81c5a78085ca","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.172604091Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125152023-12-10T18:54:49.171Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.172840581Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125162023-12-10T18:54:49.171Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.172918104Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125172023-12-10T18:54:49.171Z{"msg":"28801844-f108-4494-b663-0e921f8fd9ee WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.173001093Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125182023-12-10T18:54:49.171Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.173029781Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125192023-12-10T18:54:49.171Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.173072935Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125202023-12-10T18:54:49.171Z{"msg":"28801844-f108-4494-b663-0e921f8fd9ee WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.173110905Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125212023-12-10T18:54:49.180Z{"msg":"[2] 28801844-f108-4494-b663-0e921f8fd9ee (075da191-b9d1-4f2d-b196-81c5a78085ca) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.173263547Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125222023-12-10T18:54:49.180Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.173367264Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125232023-12-10T18:54:49.180Z{"msg":"[2] client is_active_req TRUE, promote! session 075da191-b9d1-4f2d-b196-81c5a78085ca","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.173459132Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125242023-12-10T18:54:49.180Z{"msg":"[1] 28801844-f108-4494-b663-0e921f8fd9ee (075da191-b9d1-4f2d-b196-81c5a78085ca) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.173707295Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125252023-12-10T18:54:49.181Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.17378376Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125262023-12-10T18:54:49.181Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:49.17382735Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125272023-12-10T18:54:49.181Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.173882715Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125282023-12-10T18:54:49.181Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.173909118Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125292023-12-10T18:54:49.181Z{"msg":"28801844-f108-4494-b663-0e921f8fd9ee WaitActive WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.173945384Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125302023-12-10T18:54:49.181Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.173977094Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125312023-12-10T18:54:49.181Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.174010932Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125322023-12-10T18:54:49.182Z{"msg":"[0] 28801844-f108-4494-b663-0e921f8fd9ee (075da191-b9d1-4f2d-b196-81c5a78085ca) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.174136253Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125332023-12-10T18:54:49.182Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.174213235Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125342023-12-10T18:54:49.182Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:49.174326454Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125352023-12-10T18:54:49.182Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.174383592Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125362023-12-10T18:54:49.182Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.174410979Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125372023-12-10T18:54:49.183Z{"msg":"28801844-f108-4494-b663-0e921f8fd9ee WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.174448894Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125382023-12-10T18:54:49.183Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.174498949Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125392023-12-10T18:54:49.183Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.174522742Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125402023-12-10T18:54:49.183Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.174586021Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125412023-12-10T18:54:49.183Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.174625033Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125422023-12-10T18:54:49.184Z{"msg":"28801844-f108-4494-b663-0e921f8fd9ee WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.1746775Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125432023-12-10T18:54:49.184Z{"msg":"[2] 28801844-f108-4494-b663-0e921f8fd9ee (075da191-b9d1-4f2d-b196-81c5a78085ca) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.175055328Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125442023-12-10T18:54:49.184Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.175140359Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125452023-12-10T18:54:49.184Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:49.175183755Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125462023-12-10T18:54:49.184Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.175252846Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125472023-12-10T18:54:49.184Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.175321969Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125482023-12-10T18:54:49.184Z{"msg":"28801844-f108-4494-b663-0e921f8fd9ee WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.175390058Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125492023-12-10T18:54:49.184Z{"msg":"[0]R flush_numbers[0..12]: [179, 179, 174, 178, 180, 149, 168, 165, 180, 165, 180, 177]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.175463725Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal"The guest has finished waiting for activation with:102
125502023-12-10T18:54:49.184ZWait for a query_work_queue command to finish before sending IO
125512023-12-10T18:54:49.184Z,"pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125522023-12-10T18:54:49.184Z{"msg":"[0]R generation[0..12]: [100, 100, 96, 99, 101, 85, 94, 92, 101, 92, 101, 98]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.17566847Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125532023-12-10T18:54:49.184Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.175758724Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125542023-12-10T18:54:49.184Z{"msg":"[1]R flush_numbers[0..12]: [179, 179, 174, 178, 180, 149, 168, 165, 180, 165, 180, 177]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.175842083Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125552023-12-10T18:54:49.184Z{"msg":"[1]R generation[0..12]: [100, 100, 96, 99, 101, 85, 94, 92, 101, 92, 101, 98]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.175885182Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125562023-12-10T18:54:49.185Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.175927863Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125572023-12-10T18:54:49.185Z{"msg":"[2]R flush_numbers[0..12]: [179, 179, 174, 178, 180, 149, 168, 165, 180, 165, 180, 177]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.175996439Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125582023-12-10T18:54:49.185Z{"msg":"[2]R generation[0..12]: [100, 100, 96, 99, 101, 85, 94, 92, 101, 92, 101, 98]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.176022449Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125592023-12-10T18:54:49.185Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.176044229Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125602023-12-10T18:54:49.185Z{"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.176085421Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125612023-12-10T18:54:49.185Z{"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.176108227Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125622023-12-10T18:54:49.185Z{"msg":"Next flush: 181","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.176127795Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125632023-12-10T18:54:49.185Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.176182751Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125642023-12-10T18:54:49.185Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.176212178Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125652023-12-10T18:54:49.185Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.176234802Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125662023-12-10T18:54:49.185Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.176254978Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125672023-12-10T18:54:49.185Z{"msg":"28801844-f108-4494-b663-0e921f8fd9ee is now active with session: 075da191-b9d1-4f2d-b196-81c5a78085ca","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.176283147Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125682023-12-10T18:54:49.186Z{"msg":"28801844-f108-4494-b663-0e921f8fd9ee Set Active after no repair"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
125692023-12-10T18:54:49.186Z,"v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.17639586Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125702023-12-10T18:54:49.186Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.17644857Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125712023-12-10T18:54:49.186Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.176472143Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125722023-12-10T18:54:49.186Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.176513346Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125732023-12-10T18:54:49.186Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.176535723Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125742023-12-10T18:54:49.186Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.176576573Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125752023-12-10T18:54:49.186Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.176598739Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125762023-12-10T18:54:49.186Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.17663915Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125772023-12-10T18:54:49.186Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.17666104Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125782023-12-10T18:54:49.186Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.176701686Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125792023-12-10T18:54:49.186Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.176723634Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125802023-12-10T18:54:49.186Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:49.176763704Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1570,"session_id":"075da191-b9d1-4f2d-b196-81c5a78085ca"}
125812023-12-10T18:54:49.186ZLoading write count information from file "/tmp/test_repair_verify.data"
125822023-12-10T18:54:49.186ZRun Repair workload
125832023-12-10T18:54:49.186Z01/30 Write block 123 len 28672 data: 14 14 12 12 11 12 11
125842023-12-10T18:54:49.186Z02/30 Read block 242 len 32768
125852023-12-10T18:54:49.195Z03/30 Write block 407 len 12288 data: 9 10 8
125862023-12-10T18:54:49.195Z04/30 Write block 460 len 24576 data: 6 5 7 7 7 10
125872023-12-10T18:54:49.195Z05/30 Read block 450 len 12288
125882023-12-10T18:54:49.195Z06/30 Read block 338 len 4096
125892023-12-10T18:54:49.195Z07/30 Flush
125902023-12-10T18:54:49.196Z08/30 Write block 427 len 36864 data: 10 9 8 8 7 6 6 5 5
125912023-12-10T18:54:49.196Z09/30 Write block 533 len 12288 data: 9 11 13
125922023-12-10T18:54:49.196Z10/30 Write block 230 len 32768 data: 14 13 12 9 12 10 10 11
125932023-12-10T18:54:49.196Z11/30 Read block 157 len 12288
125942023-12-10T18:54:49.199Z12/30 Write block 330 len 28672 data: 8 8 10 10 8 8 8
125952023-12-10T18:54:49.199Z13/30 Write block 591 len 32768 data: 9 8 7 6 5 5 5 3
125962023-12-10T18:54:49.199Z14/30 Write block 86 len 40960 data: 13 10 11 9 8 7 7 9 9 8
125972023-12-10T18:54:49.199Z15/30 Read block 174 len 8192
125982023-12-10T18:54:49.199Z16/30 Write block 318 len 16384 data: 4 6 7 9
125992023-12-10T18:54:49.200Z17/30 Read block 391 len 4096
126002023-12-10T18:54:49.200Z18/30 Write block 425 len 8192 data: 8 7
126012023-12-10T18:54:49.203Z19/30 Write block 363 len 16384 data: 7 6 5 5
126022023-12-10T18:54:49.203Z20/30 Read block 239 len 40960
126032023-12-10T18:54:49.206Z21/30 Read block 201 len 16384
126042023-12-10T18:54:49.209Z22/30 Write block 260 len 36864 data: 4 5 7 9 9 10 10 8 9
126052023-12-10T18:54:49.209Z23/30 Read block 448 len 8192
126062023-12-10T18:54:49.212Z24/30 Write block 276 len 8192 data: 12 12
126072023-12-10T18:54:49.212Z25/30 Write block 193 len 24576 data: 10 10 14 14 15 14
126082023-12-10T18:54:49.212Z26/30 Write block 107 len 24576 data: 7 9 8 8 7 7
126092023-12-10T18:54:49.220Z27/30 Write block 352 len 4096 data: 11
126102023-12-10T18:54:49.220Z28/30 Write block 71 len 28672 data: 8 7 8 10 9 9 10
126112023-12-10T18:54:49.220Z29/30 Write block 481 len 28672 data: 3 4 5 5 5 7 7
126122023-12-10T18:54:49.220Z30/30 Write block 404 len 36864 data: 9 10 9 10 11 9 8 6 4
126132023-12-10T18:54:49.221Z----------------------------------------------------------------
126142023-12-10T18:54:49.222Z Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:23
126152023-12-10T18:54:49.222ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
126162023-12-10T18:54:49.222Z 8 Acked 1007 Write 9 Done Done Done false
126172023-12-10T18:54:49.222Z 9 Acked 1008 Write 3 Done Done Done false
126182023-12-10T18:54:49.222Z 10 Acked 1009 Write 8 Done Done Sent false
126192023-12-10T18:54:49.222Z 11 Acked 1010 Read 3 Done Done Sent false
126202023-12-10T18:54:49.222Z 12 Acked 1011 Write 7 Done Done Sent false
126212023-12-10T18:54:49.222Z 13 Acked 1012 Write 8 Done Done Sent false
126222023-12-10T18:54:49.223Z 14 Acked 1013 Write 10 Done Done Sent false
126232023-12-10T18:54:49.223Z 15 Acked 1014 Read 2 Done Done Sent false
126242023-12-10T18:54:49.223Z 16 Acked 1015 Write 4 Done Done Sent false
126252023-12-10T18:54:49.223Z 17 Acked 1016 Read 1 Done Done Sent false
126262023-12-10T18:54:49.223Z 18 Acked 1017 Write 2 Done Done Sent false
126272023-12-10T18:54:49.224Z 19 Acked 1018 Write 4 Done Done Sent false
126282023-12-10T18:54:49.224Z 20 Acked 1019 Read 10 Done Done Sent false
126292023-12-10T18:54:49.224Z 21 Acked 1020 Read 4 Done Done Sent false
126302023-12-10T18:54:49.224Z 22 Acked 1021 Write 9 Done Done Sent false
126312023-12-10T18:54:49.224Z 23 Acked 1022 Read 2 Done Done Sent false
126322023-12-10T18:54:49.224Z 24 Acked 1023 Write 2 Done Done Sent false
126332023-12-10T18:54:49.224Z 25 Acked 1024 Write 6 Done Done Sent false
126342023-12-10T18:54:49.224Z 26 Acked 1025 Write 6 Done Done Sent false
126352023-12-10T18:54:49.224Z 27 Acked 1026 Write 1 Done Done Sent false
126362023-12-10T18:54:49.224Z 28 Acked 1027 Write 7 Done Done Sent false
126372023-12-10T18:54:49.225Z 29 Acked 1028 Write 7 Done Done Sent false
126382023-12-10T18:54:49.225Z 30 Acked 1029 Write 9 Sent Sent Sent false
126392023-12-10T18:54:49.225Z STATES DS:0 DS:1 DS:2 TOTAL
126402023-12-10T18:54:49.225Z New 0 0 0 0
126412023-12-10T18:54:49.225Z Sent 1 1 21 23
126422023-12-10T18:54:49.225Z Done 22 22 2 46
126432023-12-10T18:54:49.225Z Skipped 0 0 0 0
126442023-12-10T18:54:49.225Z Error 0 0 0 0
126452023-12-10T18:54:49.225ZLast Flush: 1006 1006 1006
126462023-12-10T18:54:49.225ZDownstairs last five completed: 1006 1005 1004 1003 1002 1001
126472023-12-10T18:54:49.226ZUpstairs last five completed: 30 29 28 27 26 25
126482023-12-10T18:54:49.226ZWrote out file "/tmp/test_repair_verify.data"
126492023-12-10T18:54:49.229Z
126502023-12-10T18:54:49.232ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
126512023-12-10T18:54:49.264ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
126522023-12-10T18:54:49.264Z 0 000-019 100 100 100 179 179 179 F F F
126532023-12-10T18:54:49.265Z 1 020-039 100 100 100 179 179 179 F F F
126542023-12-10T18:54:49.265Z 2 040-059 96 96 96 174 174 174 F F F
126552023-12-10T18:54:49.265Z 3 060-079 99 99 99 178 178 178 T T F <---
126562023-12-10T18:54:49.265Z 4 080-099 101 101 101 180 180 180 T T F <---
126572023-12-10T18:54:49.265Z 5 100-119 85 85 85 149 149 149 T T F <---
126582023-12-10T18:54:49.265Z 6 120-139 102 102 102 181 181 181 F F F
126592023-12-10T18:54:49.268Z 7 140-159 92 92 92 165 165 165 F F F
126602023-12-10T18:54:49.269Z 8 160-179 101 101 101 180 180 180 F F F
126612023-12-10T18:54:49.270Z 9 180-199 92 92 92 165 165 165 T T F <---
126622023-12-10T18:54:49.270Z 10 200-219 101 101 101 180 180 180 F F F
126632023-12-10T18:54:49.270Z 11 220-239 98 98 98 177 177 177 T T F <---
126642023-12-10T18:54:49.270Z 12 240-259 99 99 99 178 178 178 F F F
126652023-12-10T18:54:49.270Z 13 260-279 101 101 101 180 180 180 T T F <---
126662023-12-10T18:54:49.270Z 14 280-299 98 98 98 176 176 176 F F F
126672023-12-10T18:54:49.270Z 15 300-319 98 98 98 176 176 176 T T F <---
126682023-12-10T18:54:49.270Z 16 320-339 91 91 91 163 163 163 T T F <---
126692023-12-10T18:54:49.270Z 17 340-359 101 101 101 180 180 180 T T F <---
126702023-12-10T18:54:49.270Z 18 360-379 101 101 101 180 180 180 T T F <---
126712023-12-10T18:54:49.271Z 19 380-399 101 101 101 180 180 180 F F F
126722023-12-10T18:54:49.271Z 20 400-419 102 102 102 181 181 181 T T F <---
126732023-12-10T18:54:49.271Z 21 420-439 95 95 95 172 172 172 T T T
126742023-12-10T18:54:49.271Z 22 440-459 96 96 96 174 174 174 F F F
126752023-12-10T18:54:49.271Z 23 460-479 102 102 102 181 181 181 F F F
126762023-12-10T18:54:49.271Z 24 480-499 101 101 101 180 180 180 T T F <---
126772023-12-10T18:54:49.271Z 25 500-519 92 92 92 165 165 165 F F F
126782023-12-10T18:54:49.271Z 26 520-539 96 96 96 173 173 173 T T T
126792023-12-10T18:54:49.271Z 27 540-559 101 101 101 180 180 180 F F F
126802023-12-10T18:54:49.271Z 28 560-579 94 94 94 171 171 171 F F F
126812023-12-10T18:54:49.272Z 29 580-599 99 99 99 178 178 178 T T F <---
126822023-12-10T18:54:49.272ZMax gen: 102, Max flush: 181
126832023-12-10T18:54:49.272ZError: Difference in extent metadata found!
126842023-12-10T18:54:49.275ZA Difference in extent metadata is expected here
126852023-12-10T18:54:49.275ZOn loop 51
126862023-12-10T18:54:49.275Z
126872023-12-10T18:54:49.275ZVerifying data now
126882023-12-10T18:54:55.709ZNo change after verify
126892023-12-10T18:54:55.709ZLoop: 51 Downstairs dump after verify (and repair):
126902023-12-10T18:54:55.732ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
126912023-12-10T18:54:55.732Z 0 000-019 100 100 100 179 179 179 F F F
126922023-12-10T18:54:55.732Z 1 020-039 100 100 100 179 179 179 F F F
126932023-12-10T18:54:55.732Z 2 040-059 96 96 96 174 174 174 F F F
126942023-12-10T18:54:55.738Z 3 060-079 103 103 103 182 182 182 F F F
126952023-12-10T18:54:55.738Z 4 080-099 103 103 103 182 182 182 F F F
126962023-12-10T18:54:55.738Z 5 100-119 103 103 103 182 182 182 F F F
126972023-12-10T18:54:55.738Z 6 120-139 102 102 102 181 181 181 F F F
126982023-12-10T18:54:55.738Z 7 140-159 92 92 92 165 165 165 F F F
126992023-12-10T18:54:55.739Z 8 160-179 101 101 101 180 180 180 F F F
127002023-12-10T18:54:55.739Z 9 180-199 103 103 103 182 182 182 F F F
127012023-12-10T18:54:55.739Z 10 200-219 101 101 101 180 180 180 F F F
127022023-12-10T18:54:55.739Z 11 220-239 103 103 103 182 182 182 F F F
127032023-12-10T18:54:55.739Z 12 240-259 99 99 99 178 178 178 F F F
127042023-12-10T18:54:55.739Z 13 260-279 103 103 103 182 182 182 F F F
127052023-12-10T18:54:55.739Z 14 280-299 98 98 98 176 176 176 F F F
127062023-12-10T18:54:55.739Z 15 300-319 103 103 103 182 182 182 F F F
127072023-12-10T18:54:55.739Z 16 320-339 103 103 103 182 182 182 F F F
127082023-12-10T18:54:55.739Z 17 340-359 103 103 103 182 182 182 F F F
127092023-12-10T18:54:55.739Z 18 360-379 103 103 103 182 182 182 F F F
127102023-12-10T18:54:55.739Z 19 380-399 101 101 101 180 180 180 F F F
127112023-12-10T18:54:55.739Z 20 400-419 103 103 103 182 182 182 F F F
127122023-12-10T18:54:55.739Z 21 420-439 103 103 103 182 182 182 F F F
127132023-12-10T18:54:55.739Z 22 440-459 96 96 96 174 174 174 F F F
127142023-12-10T18:54:55.739Z 23 460-479 102 102 102 181 181 181 F F F
127152023-12-10T18:54:55.740Z 24 480-499 103 103 103 182 182 182 F F F
127162023-12-10T18:54:55.740Z 25 500-519 92 92 92 165 165 165 F F F
127172023-12-10T18:54:55.740Z 26 520-539 103 103 103 182 182 182 F F F
127182023-12-10T18:54:55.740Z 27 540-559 101 101 101 180 180 180 F F F
127192023-12-10T18:54:55.740Z 28 560-579 94 94 94 171 171 171 F F F
127202023-12-10T18:54:55.740Z 29 580-599 103 103 103 182 182 182 F F F
127212023-12-10T18:54:55.741ZMax gen: 103, Max flush: 182
127222023-12-10T18:54:55.744Z
127232023-12-10T18:54:55.744ZBegin loop 52
127242023-12-10T18:54:55.744ZDownstairs to restart: 1
127252023-12-10T18:54:55.766Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.772059113Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578}
127262023-12-10T18:54:55.769Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.77249265Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578}
127272023-12-10T18:54:55.770Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.772617129Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578}
127282023-12-10T18:54:55.770Z{"msg":"Crucible 2961cbe5-852f-4cc9-bf0e-1497bd689459 has session id: 26807d55-f379-4877-8e1d-590fba5f8f05","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.772659601Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127292023-12-10T18:54:55.770Z{"msg":"Crucible runtime is spawned
127302023-12-10T18:54:55.770ZUpstairs opts: Upstairs UUID: 2961cbe5-852f-4cc9-bf0e-1497bd689459, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.772751802Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127312023-12-10T18:54:55.770Z{"msg":"Crucible stats registered with UUID: 2961cbe5-852f-4cc9-bf0e-1497bd689459","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.772829062Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127322023-12-10T18:54:55.770Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30The guest has requested activation with gen:104
127332023-12-10T18:54:55.770Z,"time":"2023-12-10T18:54:55.772922338Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05","task":"up_listen"}
127342023-12-10T18:54:55.770Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.77315614Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127352023-12-10T18:54:55.770Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.773204003Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"looper":"0","session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127362023-12-10T18:54:55.771Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.773252534Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127372023-12-10T18:54:55.771Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.773316872Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"looper":"1","session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127382023-12-10T18:54:55.771Z{"msg":"[0] 2961cbe5-852f-4cc9-bf0e-1497bd689459 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.773352911Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"looper":"0","session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127392023-12-10T18:54:55.771Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.773376022Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"looper":"2","session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127402023-12-10T18:54:55.771Z{"msg":"[1] 2961cbe5-852f-4cc9-bf0e-1497bd689459 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.773523584Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"looper":"1","session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127412023-12-10T18:54:55.771Z{"msg":"[2] 2961cbe5-852f-4cc9-bf0e-1497bd689459 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.773611462Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"looper":"2","session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127422023-12-10T18:54:55.771Z{"msg":"2961cbe5-852f-4cc9-bf0e-1497bd689459 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.774117479Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127432023-12-10T18:54:55.771Z{"msg":"Set desired generation to :104","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.77417307Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127442023-12-10T18:54:55.771Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.774385522Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127452023-12-10T18:54:55.771Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.774846571Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127462023-12-10T18:54:55.771Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.774937815Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127472023-12-10T18:54:55.771Z{"msg":"[0] 2961cbe5-852f-4cc9-bf0e-1497bd689459 (26807d55-f379-4877-8e1d-590fba5f8f05) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.775117657Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127482023-12-10T18:54:55.772Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.775181597Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127492023-12-10T18:54:55.772Z{"msg":"[0] client is_active_req TRUE, promote! session 26807d55-f379-4877-8e1d-590fba5f8f05","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.775207593Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127502023-12-10T18:54:55.772Z{"msg":"[2] 2961cbe5-852f-4cc9-bf0e-1497bd689459 (26807d55-f379-4877-8e1d-590fba5f8f05) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.775232817Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127512023-12-10T18:54:55.772Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.775300491Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127522023-12-10T18:54:55.772Z{"msg":"[2] client is_active_req TRUE, promote! session 26807d55-f379-4877-8e1d-590fba5f8f05","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.775332205Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127532023-12-10T18:54:55.778Z{"msg":"[1] 2961cbe5-852f-4cc9-bf0e-1497bd689459 (26807d55-f379-4877-8e1d-590fba5f8f05) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.77583639Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127542023-12-10T18:54:55.778Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.775918526Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127552023-12-10T18:54:55.778Z{"msg":"[1] client is_active_req TRUE, promote! session 26807d55-f379-4877-8e1d-590fba5f8f05","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.775944873Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127562023-12-10T18:54:55.778Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.776025436Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127572023-12-10T18:54:55.781Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.776047726Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127582023-12-10T18:54:55.781Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.776073071Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127592023-12-10T18:54:55.781Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.776098347Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127602023-12-10T18:54:55.781Z{"msg":"2961cbe5-852f-4cc9-bf0e-1497bd689459 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.776220345Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127612023-12-10T18:54:55.781Z{"msg":"2961cbe5-852f-4cc9-bf0e-1497bd689459 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.776407499Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127622023-12-10T18:54:55.781Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.776622807Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127632023-12-10T18:54:55.781Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.776713277Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127642023-12-10T18:54:55.781Z{"msg":"2961cbe5-852f-4cc9-bf0e-1497bd689459 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.776797259Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127652023-12-10T18:54:55.781Z{"msg":"[2] 2961cbe5-852f-4cc9-bf0e-1497bd689459 (26807d55-f379-4877-8e1d-590fba5f8f05) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.77688553Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127662023-12-10T18:54:55.782Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.776930194Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127672023-12-10T18:54:55.782Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:55.777000068Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127682023-12-10T18:54:55.782Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.777040069Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127692023-12-10T18:54:55.782Z{"msg":"[0] 2961cbe5-852f-4cc9-bf0e-1497bd689459 (26807d55-f379-4877-8e1d-590fba5f8f05) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.777081507Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127702023-12-10T18:54:55.782Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.777149178Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127712023-12-10T18:54:55.782Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:55.777218802Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127722023-12-10T18:54:55.783Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.77729079Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127732023-12-10T18:54:55.783Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.777331123Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127742023-12-10T18:54:55.783Z{"msg":"2961cbe5-852f-4cc9-bf0e-1497bd689459 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.777400191Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127752023-12-10T18:54:55.783Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.777468129Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127762023-12-10T18:54:55.783Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.777506875Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127772023-12-10T18:54:55.784Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.777574086Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127782023-12-10T18:54:55.784Z{"msg":"2961cbe5-852f-4cc9-bf0e-1497bd689459 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.77765966Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127792023-12-10T18:54:55.784Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.777702494Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127802023-12-10T18:54:55.784Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.777741205Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127812023-12-10T18:54:55.784Z{"msg":"[1] 2961cbe5-852f-4cc9-bf0e-1497bd689459 (26807d55-f379-4877-8e1d-590fba5f8f05) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.777778409Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127822023-12-10T18:54:55.784Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.777830314Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127832023-12-10T18:54:55.784Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:54:55.77785468Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127842023-12-10T18:54:55.784Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.777874635Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127852023-12-10T18:54:55.784Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.777922863Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127862023-12-10T18:54:55.784Z{"msg":"2961cbe5-852f-4cc9-bf0e-1497bd689459 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:104
127872023-12-10T18:54:55.790Z,"time":"2023-12-10T18:54:55.777946155Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","Wait for a query_work_queue command to finish before sending IO
127882023-12-10T18:54:55.790Zpid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127892023-12-10T18:54:55.790Z{"msg":"[0]R flush_numbers[0..12]: [179, 179, 174, 182, 182, 182, 181, 165, 180, 182, 180, 182]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.777977758Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127902023-12-10T18:54:55.790Z{"msg":"[0]R generation[0..12]: [100, 100, 96, 103, 103, 103, 102, 92, 101, 103, 101, 103]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.777999394Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127912023-12-10T18:54:55.790Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.778032442Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127922023-12-10T18:54:55.790Z{"msg":"[1]R flush_numbers[0..12]: [179, 179, 174, 182, 182, 182, 181, 165, 180, 182, 180, 182]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.778096364Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127932023-12-10T18:54:55.790Z{"msg":"[1]R generation[0..12]: [100, 100, 96, 103, 103, 103, 102, 92, 101, 103, 101, 103]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.778211079Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127942023-12-10T18:54:55.790Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.778254998Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127952023-12-10T18:54:55.790Z{"msg":"[2]R flush_numbers[0..12]: [179, 179, 174, 182, 182, 182, 181, 165, 180, 182, 180, 182]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.778357362Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127962023-12-10T18:54:55.790Z{"msg":"[2]R generation[0..12]: [100, 100, 96, 103, 103, 103, 102, 92, 101, 103, 101, 103]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.778389954Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127972023-12-10T18:54:55.790Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.778412317Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127982023-12-10T18:54:55.790Z{"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.778433524Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
127992023-12-10T18:54:55.790Z{"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.778453615Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
128002023-12-10T18:54:55.790Z{"msg":"Next flush: 183","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.778472924Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
128012023-12-10T18:54:55.791Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.778515014Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
128022023-12-10T18:54:55.791Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.778537124Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
128032023-12-10T18:54:55.791Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.778556807Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
128042023-12-10T18:54:55.791Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.778576635Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
128052023-12-10T18:54:55.791Z{"msg":"2961cbe5-852f-4cc9-bf0e-1497bd689459 is now active with session: 26807d55-f379-4877-8e1d-590fba5f8f05","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.778609154Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
128062023-12-10T18:54:55.791Z{"msg":"2961cbe5-852f-4cc9-bf0e-1497bd689459 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.778668151Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
128072023-12-10T18:54:55.791Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.77870789Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
128082023-12-10T18:54:55.791Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.778770294Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
128092023-12-10T18:54:55.792Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.778809448Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
128102023-12-10T18:54:55.792Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.778865093Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
128112023-12-10T18:54:55.792Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.778903013Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
128122023-12-10T18:54:55.792Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
128132023-12-10T18:54:55.792Z[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.779007511Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
128142023-12-10T18:54:55.792Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.779068446Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
128152023-12-10T18:54:55.792Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.779107977Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
128162023-12-10T18:54:55.793Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.779160988Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
128172023-12-10T18:54:55.793Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.779199774Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
128182023-12-10T18:54:55.793Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:54:55.779253114Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1578,"session_id":"26807d55-f379-4877-8e1d-590fba5f8f05"}
128192023-12-10T18:54:55.793ZLoading write count information from file "/tmp/test_repair_verify.data"
128202023-12-10T18:54:55.793ZRun Repair workload
128212023-12-10T18:54:55.793Z01/30 Write block 571 len 28672 data: 9 7 7 8 9 10 10
128222023-12-10T18:54:55.797Z02/30 Write block 497 len 36864 data: 5 7 7 10 11 11 12 8 7
128232023-12-10T18:54:55.797Z03/30 Read block 333 len 28672
128242023-12-10T18:54:55.797Z04/30 Write block 467 len 4096 data: 9
128252023-12-10T18:54:55.797Z05/30 Read block 403 len 12288
128262023-12-10T18:54:55.797Z06/30 Read block 88 len 20480
128272023-12-10T18:54:55.797Z07/30 Read block 263 len 36864
128282023-12-10T18:54:55.797Z08/30 Read block 328 len 20480
128292023-12-10T18:54:55.798Z09/30 Read block 581 len 20480
128302023-12-10T18:54:55.798Z10/30 Read block 195 len 28672
128312023-12-10T18:54:55.798Z11/30 Read block 221 len 16384
128322023-12-10T18:54:55.798Z12/30 Write block 477 len 32768 data: 4 3 4 3 4 5 6 6
128332023-12-10T18:54:55.800Z13/30 Read block 378 len 36864
128342023-12-10T18:54:55.800Z14/30 Read block 404 len 4096
128352023-12-10T18:54:55.804Z15/30 Write block 318 len 24576 data: 5 7 8 10 11 10
128362023-12-10T18:54:55.804Z16/30 Write block 209 len 12288 data: 6 5 6
128372023-12-10T18:54:55.806Z17/30 Flush
128382023-12-10T18:54:55.809Z18/30 Write block 219 len 16384 data: 11 11 14 13
128392023-12-10T18:54:55.809Z19/30 Read block 415 len 12288
128402023-12-10T18:54:55.812Z20/30 Read block 289 len 28672
128412023-12-10T18:54:55.814Z21/30 Write block 217 len 12288 data: 14 13 12
128422023-12-10T18:54:55.814Z22/30 Read block 318 len 8192
128432023-12-10T18:54:55.817Z23/30 Write block 40 len 28672 data: 8 6 6 5 5 7 6
128442023-12-10T18:54:55.817Z24/30 Read block 358 len 12288
128452023-12-10T18:54:55.820Z25/30 Write block 318 len 12288 data: 6 8 9
128462023-12-10T18:54:55.820Z26/30 Write block 190 len 8192 data: 8 9
128472023-12-10T18:54:55.820Z27/30 Write block 419 len 36864 data: 8 7 9 8 8 11 9 8 11
128482023-12-10T18:54:55.820Z28/30 Read block 461 len 32768
128492023-12-10T18:54:55.823Z29/30 Read block 176 len 20480
128502023-12-10T18:54:55.826Z30/30 Write block 69 len 20480 data: 8 7 9 8 9
128512023-12-10T18:54:55.832Z----------------------------------------------------------------
128522023-12-10T18:54:55.832Z Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30
128532023-12-10T18:54:55.832ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
128542023-12-10T18:54:55.832Z 1 Acked 1000 Write 7 Done Done Done false
128552023-12-10T18:54:55.832Z 2 Acked 1001 Write 9 Done Sent Done false
128562023-12-10T18:54:55.832Z 3 Acked 1002 Read 7 Done Sent Done false
128572023-12-10T18:54:55.832Z 4 Acked 1003 Write 1 Done Sent Done false
128582023-12-10T18:54:55.832Z 5 Acked 1004 Read 3 Done Sent Done false
128592023-12-10T18:54:55.832Z 6 Acked 1005 Read 5 Done Sent Done false
128602023-12-10T18:54:55.833Z 7 Acked 1006 Read 9 Done Sent Done false
128612023-12-10T18:54:55.833Z 8 Acked 1007 Read 5 Done Sent Done false
128622023-12-10T18:54:55.833Z 9 Acked 1008 Read 5 Done Sent Done false
128632023-12-10T18:54:55.833Z 10 Acked 1009 Read 7 Done Sent Done false
128642023-12-10T18:54:55.833Z 11 Acked 1010 Read 4 Done Sent Done false
128652023-12-10T18:54:55.833Z 12 Acked 1011 Write 8 Done Sent Done false
128662023-12-10T18:54:55.833Z 13 Acked 1012 Read 9 Done Sent Done false
128672023-12-10T18:54:55.833Z 14 Acked 1013 Read 1 Done Sent Done false
128682023-12-10T18:54:55.833Z 15 Acked 1014 Write 6 Done Sent Done false
128692023-12-10T18:54:55.833Z 16 Acked 1015 Write 3 Done Sent Done false
128702023-12-10T18:54:55.833Z 17 Acked 1016 Flush 0 Done Sent Done false
128712023-12-10T18:54:55.834Z 18 Acked 1017 Write 4 Done Sent Done false
128722023-12-10T18:54:55.834Z 19 Acked 1018 Read 3 Done Sent Done false
128732023-12-10T18:54:55.834Z 20 Acked 1019 Read 7 Done Sent Done false
128742023-12-10T18:54:55.834Z 21 Acked 1020 Write 3 Done Sent Done false
128752023-12-10T18:54:55.834Z 22 Acked 1021 Read 2 Done Sent Done false
128762023-12-10T18:54:55.834Z 23 Acked 1022 Write 7 Done Sent Done false
128772023-12-10T18:54:55.834Z 24 Acked 1023 Read 3 Done Sent Done false
128782023-12-10T18:54:55.834Z 25 Acked 1024 Write 3 Done Sent Done false
128792023-12-10T18:54:55.834Z 26 Acked 1025 Write 2 Done Sent Done false
128802023-12-10T18:54:55.834Z 27 Acked 1026 Write 9 Done Sent Done false
128812023-12-10T18:54:55.834Z 28 Acked 1027 Read 8 Done Sent Done false
128822023-12-10T18:54:55.834Z 29 Acked 1028 Read 5 Done Sent Done false
128832023-12-10T18:54:55.835Z 30 Acked 1029 Write 5 Sent Sent Sent false
128842023-12-10T18:54:55.835Z STATES DS:0 DS:1 DS:2 TOTAL
128852023-12-10T18:54:55.835Z New 0 0 0 0
128862023-12-10T18:54:55.835Z Sent 1 29 1 31
128872023-12-10T18:54:55.835Z Done 29 1 29 59
128882023-12-10T18:54:55.835Z Skipped 0 0 0 0
128892023-12-10T18:54:55.835Z Error 0 0 0 0
128902023-12-10T18:54:55.835ZLast Flush: 1016 0 1016
128912023-12-10T18:54:55.835ZDownstairs last five completed:
128922023-12-10T18:54:55.835ZUpstairs last five completed: 30 29 28 27 26 25
128932023-12-10T18:54:55.835ZWrote out file "/tmp/test_repair_verify.data"
128942023-12-10T18:54:55.838Z
128952023-12-10T18:54:55.841ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
128962023-12-10T18:54:55.868ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
128972023-12-10T18:54:55.868Z 0 000-019 100 100 100 179 179 179 F F F
128982023-12-10T18:54:55.871Z 1 020-039 100 100 100 179 179 179 F F F
128992023-12-10T18:54:55.871Z 2 040-059 96 96 96 174 174 174 T F T <---
129002023-12-10T18:54:55.871Z 3 060-079 103 103 103 182 182 182 T F T <---
129012023-12-10T18:54:55.871Z 4 080-099 103 103 103 182 182 182 F F F
129022023-12-10T18:54:55.871Z 5 100-119 103 103 103 182 182 182 F F F
129032023-12-10T18:54:55.871Z 6 120-139 102 102 102 181 181 181 F F F
129042023-12-10T18:54:55.871Z 7 140-159 92 92 92 165 165 165 F F F
129052023-12-10T18:54:55.871Z 8 160-179 101 101 101 180 180 180 F F F
129062023-12-10T18:54:55.872Z 9 180-199 103 103 103 182 182 182 T F T <---
129072023-12-10T18:54:55.872Z 10 200-219 104 101 104 183 180 183 T F T <---
129082023-12-10T18:54:55.872Z 11 220-239 103 103 103 182 182 182 T F T <---
129092023-12-10T18:54:55.872Z 12 240-259 99 99 99 178 178 178 F F F
129102023-12-10T18:54:55.872Z 13 260-279 103 103 103 182 182 182 F F F
129112023-12-10T18:54:55.872Z 14 280-299 98 98 98 176 176 176 F F F
129122023-12-10T18:54:55.872Z 15 300-319 104 103 104 183 182 183 T F T <---
129132023-12-10T18:54:55.872Z 16 320-339 104 103 104 183 182 183 T F T <---
129142023-12-10T18:54:55.872Z 17 340-359 103 103 103 182 182 182 F F F
129152023-12-10T18:54:55.872Z 18 360-379 103 103 103 182 182 182 F F F
129162023-12-10T18:54:55.872Z 19 380-399 101 101 101 180 180 180 F F F
129172023-12-10T18:54:55.872Z 20 400-419 103 103 103 182 182 182 T F T <---
129182023-12-10T18:54:55.872Z 21 420-439 103 103 103 182 182 182 T F T <---
129192023-12-10T18:54:55.872Z 22 440-459 96 96 96 174 174 174 F F F
129202023-12-10T18:54:55.873Z 23 460-479 104 102 104 183 181 183 F F F <---
129212023-12-10T18:54:55.873Z 24 480-499 104 103 104 183 182 183 F F F <---
129222023-12-10T18:54:55.873Z 25 500-519 104 92 104 183 165 183 F F F <---
129232023-12-10T18:54:55.873Z 26 520-539 103 103 103 182 182 182 F F F
129242023-12-10T18:54:55.873Z 27 540-559 101 101 101 180 180 180 F F F
129252023-12-10T18:54:55.873Z 28 560-579 104 94 104 183 171 183 F T F <---
129262023-12-10T18:54:55.873Z 29 580-599 103 103 103 182 182 182 F F F
129272023-12-10T18:54:55.873ZMax gen: 104, Max flush: 183
129282023-12-10T18:54:55.873ZError: Difference in extent metadata found!
129292023-12-10T18:54:55.876ZA Difference in extent metadata is expected here
129302023-12-10T18:54:55.877ZOn loop 52
129312023-12-10T18:54:55.877Z
129322023-12-10T18:54:55.877ZVerifying data now
129332023-12-10T18:55:01.156ZNo change after verify
129342023-12-10T18:55:01.156ZLoop: 52 Downstairs dump after verify (and repair):
129352023-12-10T18:55:01.178ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
129362023-12-10T18:55:01.178Z 0 000-019 100 100 100 179 179 179 F F F
129372023-12-10T18:55:01.182Z 1 020-039 100 100 100 179 179 179 F F F
129382023-12-10T18:55:01.182Z 2 040-059 105 105 105 184 184 184 F F F
129392023-12-10T18:55:01.182Z 3 060-079 105 105 105 184 184 184 F F F
129402023-12-10T18:55:01.182Z 4 080-099 103 103 103 182 182 182 F F F
129412023-12-10T18:55:01.182Z 5 100-119 103 103 103 182 182 182 F F F
129422023-12-10T18:55:01.182Z 6 120-139 102 102 102 181 181 181 F F F
129432023-12-10T18:55:01.182Z 7 140-159 92 92 92 165 165 165 F F F
129442023-12-10T18:55:01.182Z 8 160-179 101 101 101 180 180 180 F F F
129452023-12-10T18:55:01.182Z 9 180-199 105 105 105 184 184 184 F F F
129462023-12-10T18:55:01.183Z 10 200-219 105 105 105 184 184 184 F F F
129472023-12-10T18:55:01.183Z 11 220-239 105 105 105 184 184 184 F F F
129482023-12-10T18:55:01.183Z 12 240-259 99 99 99 178 178 178 F F F
129492023-12-10T18:55:01.183Z 13 260-279 103 103 103 182 182 182 F F F
129502023-12-10T18:55:01.183Z 14 280-299 98 98 98 176 176 176 F F F
129512023-12-10T18:55:01.183Z 15 300-319 105 105 105 184 184 184 F F F
129522023-12-10T18:55:01.183Z 16 320-339 105 105 105 184 184 184 F F F
129532023-12-10T18:55:01.183Z 17 340-359 103 103 103 182 182 182 F F F
129542023-12-10T18:55:01.183Z 18 360-379 103 103 103 182 182 182 F F F
129552023-12-10T18:55:01.183Z 19 380-399 101 101 101 180 180 180 F F F
129562023-12-10T18:55:01.183Z 20 400-419 105 105 105 184 184 184 F F F
129572023-12-10T18:55:01.183Z 21 420-439 105 105 105 184 184 184 F F F
129582023-12-10T18:55:01.183Z 22 440-459 96 96 96 174 174 174 F F F
129592023-12-10T18:55:01.183Z 23 460-479 104 104 104 183 183 183 F F F
129602023-12-10T18:55:01.183Z 24 480-499 104 104 104 183 183 183 F F F
129612023-12-10T18:55:01.183Z 25 500-519 104 104 104 183 183 183 F F F
129622023-12-10T18:55:01.183Z 26 520-539 103 103 103 182 182 182 F F F
129632023-12-10T18:55:01.184Z 27 540-559 101 101 101 180 180 180 F F F
129642023-12-10T18:55:01.184Z 28 560-579 104 104 104 183 183 183 F F F
129652023-12-10T18:55:01.184Z 29 580-599 103 103 103 182 182 182 F F F
129662023-12-10T18:55:01.184ZMax gen: 105, Max flush: 184
129672023-12-10T18:55:01.188Z
129682023-12-10T18:55:01.188ZBegin loop 53
129692023-12-10T18:55:01.188ZDownstairs to restart: 1
129702023-12-10T18:55:01.214Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.219563955Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586}
129712023-12-10T18:55:01.218Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.219985997Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586}
129722023-12-10T18:55:01.218Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.220051976Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586}
129732023-12-10T18:55:01.218Z{"msg":"Crucible f5348177-2d37-496f-99a7-b1b5bc8dcad0 has session id: e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.220076676Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
129742023-12-10T18:55:01.218Z{"msg":"Upstairs opts: Upstairs UUID: f5348177-2d37-496f-99a7-b1b5bc8dcad0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.220125277Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
129752023-12-10T18:55:01.218Z{"msg":"Crucible stats registered with UUID: f5348177-2d37-496f-99a7-b1b5bc8dcad0","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.220169063Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
129762023-12-10T18:55:01.218ZCrucible runtime is spawned
129772023-12-10T18:55:01.218ZThe guest has requested activation with gen:106
129782023-12-10T18:55:01.219Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.220510949Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f","task":"up_listen"}
129792023-12-10T18:55:01.219Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.220592819Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"looper":"2","session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
129802023-12-10T18:55:01.219Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.220644954Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
129812023-12-10T18:55:01.219Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.22071419Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
129822023-12-10T18:55:01.219Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.220959778Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"looper":"1","session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
129832023-12-10T18:55:01.219Z{"msg":"[2] f5348177-2d37-496f-99a7-b1b5bc8dcad0 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.221016638Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"looper":"2","session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
129842023-12-10T18:55:01.220Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.221108441Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"looper":"0","session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
129852023-12-10T18:55:01.220Z{"msg":"[0] f5348177-2d37-496f-99a7-b1b5bc8dcad0 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.221297911Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"looper":"0","session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
129862023-12-10T18:55:01.220Z{"msg":"[1] f5348177-2d37-496f-99a7-b1b5bc8dcad0 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.221364438Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"looper":"1","session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
129872023-12-10T18:55:01.220Z{"msg":"f5348177-2d37-496f-99a7-b1b5bc8dcad0 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.221788669Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
129882023-12-10T18:55:01.220Z{"msg":"Set desired generation to :106","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.22185796Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
129892023-12-10T18:55:01.220Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.221973514Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
129902023-12-10T18:55:01.220Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.222414462Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
129912023-12-10T18:55:01.220Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.222555194Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
129922023-12-10T18:55:01.220Z{"msg":"[0] f5348177-2d37-496f-99a7-b1b5bc8dcad0 (e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.222949303Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
129932023-12-10T18:55:01.220Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.223029932Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
129942023-12-10T18:55:01.220Z{"msg":"[0] client is_active_req TRUE, promote! session e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.223110718Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
129952023-12-10T18:55:01.220Z{"msg":"[2] f5348177-2d37-496f-99a7-b1b5bc8dcad0 (e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.223282695Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
129962023-12-10T18:55:01.220Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.223326389Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
129972023-12-10T18:55:01.220Z{"msg":"[2] client is_active_req TRUE, promote! session e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.223399796Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
129982023-12-10T18:55:01.220Z{"msg":"[1] f5348177-2d37-496f-99a7-b1b5bc8dcad0 (e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.223433232Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
129992023-12-10T18:55:01.229Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.223454309Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130002023-12-10T18:55:01.229Z{"msg":"[1] client is_active_req TRUE, promote! session e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.223505617Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130012023-12-10T18:55:01.229Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.22385959Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130022023-12-10T18:55:01.229Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.223948766Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130032023-12-10T18:55:01.229Z{"msg":"f5348177-2d37-496f-99a7-b1b5bc8dcad0 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.223993169Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130042023-12-10T18:55:01.229Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.224099474Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130052023-12-10T18:55:01.229Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.224202657Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130062023-12-10T18:55:01.229Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.224245761Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130072023-12-10T18:55:01.229Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.2242913Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130082023-12-10T18:55:01.230Z{"msg":"f5348177-2d37-496f-99a7-b1b5bc8dcad0 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.224351941Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130092023-12-10T18:55:01.230Z{"msg":"f5348177-2d37-496f-99a7-b1b5bc8dcad0 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.224466329Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130102023-12-10T18:55:01.230Z{"msg":"[0] f5348177-2d37-496f-99a7-b1b5bc8dcad0 (e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.224812921Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130112023-12-10T18:55:01.231Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.224878927Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130122023-12-10T18:55:01.231Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:01.224903819Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130132023-12-10T18:55:01.231Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.225026694Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130142023-12-10T18:55:01.231Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.225096414Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130152023-12-10T18:55:01.231Z{"msg":"f5348177-2d37-496f-99a7-b1b5bc8dcad0 WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.225140302Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130162023-12-10T18:55:01.231Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.225201368Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130172023-12-10T18:55:01.231Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.225277175Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130182023-12-10T18:55:01.232Z{"msg":"[1] f5348177-2d37-496f-99a7-b1b5bc8dcad0 (e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.225321956Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130192023-12-10T18:55:01.232Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.225379617Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130202023-12-10T18:55:01.232Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:01.225420137Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130212023-12-10T18:55:01.232Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.225476565Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130222023-12-10T18:55:01.232Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.225517767Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130232023-12-10T18:55:01.232Z{"msg":"f5348177-2d37-496f-99a7-b1b5bc8dcad0 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.22558072Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130242023-12-10T18:55:01.233Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.225622251Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130252023-12-10T18:55:01.233Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.225678179Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130262023-12-10T18:55:01.233Z{"msg":"[2] f5348177-2d37-496f-99a7-b1b5bc8dcad0 (e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.225779079Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_idThe guest has finished waiting for activation with:106
130272023-12-10T18:55:01.233Z":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130282023-12-10T18:55:01.233ZWait for a query_work_queue command to finish before sending IO
130292023-12-10T18:55:01.233Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.22582336Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130302023-12-10T18:55:01.233Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:01.225896312Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130312023-12-10T18:55:01.233Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.225936446Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130322023-12-10T18:55:01.233Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.225976842Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130332023-12-10T18:55:01.233Z{"msg":"f5348177-2d37-496f-99a7-b1b5bc8dcad0 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226009418Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130342023-12-10T18:55:01.233Z{"msg":"[0]R flush_numbers[0..12]: [179, 179, 184, 184, 182, 182, 181, 165, 180, 184, 184, 184]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226038947Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130352023-12-10T18:55:01.233Z{"msg":"[0]R generation[0..12]: [100, 100, 105, 105, 103, 103, 102, 92, 101, 105, 105, 105]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226096337Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130362023-12-10T18:55:01.233Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226122437Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130372023-12-10T18:55:01.233Z{"msg":"[1]R flush_numbers[0..12]: [179, 179, 184, 184, 182, 182, 181, 165, 180, 184, 184, 184]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226184641Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130382023-12-10T18:55:01.233Z{"msg":"[1]R generation[0..12]: [100, 100, 105, 105, 103, 103, 102, 92, 101, 105, 105, 105]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226227825Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130392023-12-10T18:55:01.233Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226291983Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130402023-12-10T18:55:01.234Z{"msg":"[2]R flush_numbers[0..12]: [179, 179, 184, 184, 182, 182, 181, 165, 180, 184, 184, 184]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226333214Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130412023-12-10T18:55:01.234Z{"msg":"[2]R generation[0..12]: [100, 100, 105, 105, 103, 103, 102, 92, 101, 105, 105, 105]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226386299Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130422023-12-10T18:55:01.234Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.22641329Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130432023-12-10T18:55:01.234Z{"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226433327Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130442023-12-10T18:55:01.235Z{"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226453738Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130452023-12-10T18:55:01.235Z{"msg":"Next flush: 185","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226503943Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130462023-12-10T18:55:01.235Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226527323Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130472023-12-10T18:55:01.235Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.22654769Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130482023-12-10T18:55:01.235Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226589815Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130492023-12-10T18:55:01.235Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226612889Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130502023-12-10T18:55:01.235Z{"msg":"f5348177-2d37-496f-99a7-b1b5bc8dcad0 is now active with session: e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226634576Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130512023-12-10T18:55:01.236Z{"msg":"f5348177-2d37-496f-99a7-b1b5bc8dcad0 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226680611Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130522023-12-10T18:55:01.236Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226703696Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130532023-12-10T18:55:01.236Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226723258Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130542023-12-10T18:55:01.236Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226763546Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130552023-12-10T18:55:01.236Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226785844Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130562023-12-10T18:55:01.236Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226805562Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130572023-12-10T18:55:01.236Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226848857Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130582023-12-10T18:55:01.236Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226870766Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130592023-12-10T18:55:01.236Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226890509Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130602023-12-10T18:55:01.236Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226919559Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130612023-12-10T18:55:01.236Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.226953327Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130622023-12-10T18:55:01.236Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:01.227008499Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1586,"session_id":"e5a17cbd-f871-4a26-bdaf-2225ad9a3c2f"}
130632023-12-10T18:55:01.236ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
130642023-12-10T18:55:01.237ZLoading write count information from file "/tmp/test_repair_verify.data"
130652023-12-10T18:55:01.237ZRun Repair workload
130662023-12-10T18:55:01.237Z01/30 Write block 79 len 24576 data: 12 11 12 12 12 12
130672023-12-10T18:55:01.241Z02/30 Read block 556 len 16384
130682023-12-10T18:55:01.241Z03/30 Read block 150 len 32768
130692023-12-10T18:55:01.241Z04/30 Write block 95 len 8192 data: 9 7
130702023-12-10T18:55:01.241Z05/30 Write block 244 len 20480 data: 13 12 13 11 12
130712023-12-10T18:55:01.241Z06/30 Read block 203 len 4096
130722023-12-10T18:55:01.241Z07/30 Write block 562 len 4096 data: 10
130732023-12-10T18:55:01.241Z08/30 Read block 541 len 12288
130742023-12-10T18:55:01.241Z09/30 Write block 267 len 24576 data: 9 10 10 9 8 8
130752023-12-10T18:55:01.241Z10/30 Read block 427 len 36864
130762023-12-10T18:55:01.244Z11/30 Read block 41 len 40960
130772023-12-10T18:55:01.245Z12/30 Flush
130782023-12-10T18:55:01.248Z13/30 Write block 523 len 40960 data: 6 7 7 7 6 6 7 8 12 12
130792023-12-10T18:55:01.251Z14/30 Write block 501 len 4096 data: 12
130802023-12-10T18:55:01.251Z15/30 Flush
130812023-12-10T18:55:01.254Z16/30 Write block 301 len 4096 data: 9
130822023-12-10T18:55:01.254Z17/30 Flush
130832023-12-10T18:55:01.257Z18/30 Write block 376 len 40960 data: 13 12 16 14 15 11 12 10 12 12
130842023-12-10T18:55:01.257Z19/30 Read block 394 len 8192
130852023-12-10T18:55:01.261Z20/30 Write block 549 len 4096 data: 7
130862023-12-10T18:55:01.262Z21/30 Read block 311 len 4096
130872023-12-10T18:55:01.262Z22/30 Write block 45 len 16384 data: 8 7 7 7
130882023-12-10T18:55:01.266Z23/30 Read block 517 len 40960
130892023-12-10T18:55:01.266Z24/30 Read block 64 len 40960
130902023-12-10T18:55:01.269Z25/30 Write block 197 len 32768 data: 16 15 13 12 14 12 14 12
130912023-12-10T18:55:01.269Z26/30 Read block 490 len 4096
130922023-12-10T18:55:01.273Z27/30 Write block 179 len 28672 data: 3 3 5 7 8 6 7
130932023-12-10T18:55:01.273Z28/30 Write block 70 len 36864 data: 8 10 9 10 11 10 10 11 10
130942023-12-10T18:55:01.276Z29/30 Read block 440 len 16384
130952023-12-10T18:55:01.276Z30/30 Read block 167 len 32768
130962023-12-10T18:55:01.280Z----------------------------------------------------------------
130972023-12-10T18:55:01.280Z Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30
130982023-12-10T18:55:01.280ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
130992023-12-10T18:55:01.280Z 1 Acked 1000 Write 6 Done Done Done false
131002023-12-10T18:55:01.280Z 2 Acked 1001 Read 4 Done Done Done false
131012023-12-10T18:55:01.280Z 3 Acked 1002 Read 8 Done Done Done false
131022023-12-10T18:55:01.280Z 4 Acked 1003 Write 2 Done Sent Done false
131032023-12-10T18:55:01.280Z 5 Acked 1004 Write 5 Done Sent Done false
131042023-12-10T18:55:01.280Z 6 Acked 1005 Read 1 Done Sent Done false
131052023-12-10T18:55:01.280Z 7 Acked 1006 Write 1 Done Sent Done false
131062023-12-10T18:55:01.280Z 8 Acked 1007 Read 3 Done Sent Done false
131072023-12-10T18:55:01.280Z 9 Acked 1008 Write 6 Done Sent Done false
131082023-12-10T18:55:01.280Z 10 Acked 1009 Read 9 Done Sent Done false
131092023-12-10T18:55:01.281Z 11 Acked 1010 Read 10 Done Sent Done false
131102023-12-10T18:55:01.282Z 12 Acked 1011 Flush 0 Done Sent Done false
131112023-12-10T18:55:01.282Z 13 Acked 1012 Write 10 Done Sent Done false
131122023-12-10T18:55:01.282Z 14 Acked 1013 Write 1 Done Sent Done false
131132023-12-10T18:55:01.282Z 15 Acked 1014 Flush 0 Done Sent Done false
131142023-12-10T18:55:01.282Z 16 Acked 1015 Write 1 Done Sent Done false
131152023-12-10T18:55:01.282Z 17 Acked 1016 Flush 0 Done Sent Done false
131162023-12-10T18:55:01.282Z 18 Acked 1017 Write 10 Done Sent Done false
131172023-12-10T18:55:01.283Z 19 Acked 1018 Read 2 Done Sent Done false
131182023-12-10T18:55:01.283Z 20 Acked 1019 Write 1 Done Sent Done false
131192023-12-10T18:55:01.283Z 21 Acked 1020 Read 1 Done Sent Done false
131202023-12-10T18:55:01.283Z 22 Acked 1021 Write 4 Done Sent Done false
131212023-12-10T18:55:01.283Z 23 Acked 1022 Read 10 Done Sent Done false
131222023-12-10T18:55:01.283Z 24 Acked 1023 Read 10 Done Sent Done false
131232023-12-10T18:55:01.283Z 25 Acked 1024 Write 8 Done Sent Done false
131242023-12-10T18:55:01.283Z 26 Acked 1025 Read 1 Done Sent Done false
131252023-12-10T18:55:01.283Z 27 Acked 1026 Write 7 Done Sent Done false
131262023-12-10T18:55:01.283Z 28 Acked 1027 Write 9 Done Sent Done false
131272023-12-10T18:55:01.283Z 29 Acked 1028 Read 4 Done Sent Done false
131282023-12-10T18:55:01.283Z 30 Acked 1029 Read 8 Done Sent Done false
131292023-12-10T18:55:01.283Z STATES DS:0 DS:1 DS:2 TOTAL
131302023-12-10T18:55:01.283Z New 0 0 0 0
131312023-12-10T18:55:01.284Z Sent 0 27 0 27
131322023-12-10T18:55:01.284Z Done 30 3 30 63
131332023-12-10T18:55:01.284Z Skipped 0 0 0 0
131342023-12-10T18:55:01.284Z Error 0 0 0 0
131352023-12-10T18:55:01.284ZLast Flush: 1016 0 1016
131362023-12-10T18:55:01.285ZDownstairs last five completed:
131372023-12-10T18:55:01.285ZUpstairs last five completed: 30 29 28 27 26 25
131382023-12-10T18:55:01.285ZWrote out file "/tmp/test_repair_verify.data"
131392023-12-10T18:55:01.289Z
131402023-12-10T18:55:01.292ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
131412023-12-10T18:55:01.317ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
131422023-12-10T18:55:01.318Z 0 000-019 100 100 100 179 179 179 F F F
131432023-12-10T18:55:01.321Z 1 020-039 100 100 100 179 179 179 F F F
131442023-12-10T18:55:01.321Z 2 040-059 105 105 105 184 184 184 T F T <---
131452023-12-10T18:55:01.321Z 3 060-079 106 105 106 185 184 185 T T T <---
131462023-12-10T18:55:01.321Z 4 080-099 106 103 106 185 182 185 F T F <---
131472023-12-10T18:55:01.322Z 5 100-119 103 103 103 182 182 182 F F F
131482023-12-10T18:55:01.322Z 6 120-139 102 102 102 181 181 181 F F F
131492023-12-10T18:55:01.322Z 7 140-159 92 92 92 165 165 165 F F F
131502023-12-10T18:55:01.322Z 8 160-179 101 101 101 180 180 180 T F T <---
131512023-12-10T18:55:01.322Z 9 180-199 105 105 105 184 184 184 T F T <---
131522023-12-10T18:55:01.322Z 10 200-219 105 105 105 184 184 184 T F T <---
131532023-12-10T18:55:01.322Z 11 220-239 105 105 105 184 184 184 F F F
131542023-12-10T18:55:01.322Z 12 240-259 106 99 106 185 178 185 F F F <---
131552023-12-10T18:55:01.322Z 13 260-279 106 103 106 185 182 185 F F F <---
131562023-12-10T18:55:01.322Z 14 280-299 98 98 98 176 176 176 F F F
131572023-12-10T18:55:01.324Z 15 300-319 106 105 106 187 184 187 F F F <---
131582023-12-10T18:55:01.324Z 16 320-339 105 105 105 184 184 184 F F F
131592023-12-10T18:55:01.324Z 17 340-359 103 103 103 182 182 182 F F F
131602023-12-10T18:55:01.324Z 18 360-379 103 103 103 182 182 182 T F T <---
131612023-12-10T18:55:01.324Z 19 380-399 101 101 101 180 180 180 T F T <---
131622023-12-10T18:55:01.324Z 20 400-419 105 105 105 184 184 184 F F F
131632023-12-10T18:55:01.324Z 21 420-439 105 105 105 184 184 184 F F F
131642023-12-10T18:55:01.324Z 22 440-459 96 96 96 174 174 174 F F F
131652023-12-10T18:55:01.324Z 23 460-479 104 104 104 183 183 183 F F F
131662023-12-10T18:55:01.324Z 24 480-499 104 104 104 183 183 183 F F F
131672023-12-10T18:55:01.324Z 25 500-519 106 104 106 186 183 186 F F F <---
131682023-12-10T18:55:01.324Z 26 520-539 106 103 106 186 182 186 F F F <---
131692023-12-10T18:55:01.324Z 27 540-559 101 101 101 180 180 180 T F T <---
131702023-12-10T18:55:01.324Z 28 560-579 106 104 106 185 183 185 F F F <---
131712023-12-10T18:55:01.324Z 29 580-599 103 103 103 182 182 182 F F F
131722023-12-10T18:55:01.324ZMax gen: 106, Max flush: 187
131732023-12-10T18:55:01.324ZError: Difference in extent metadata found!
131742023-12-10T18:55:01.331ZA Difference in extent metadata is expected here
131752023-12-10T18:55:01.331ZOn loop 53
131762023-12-10T18:55:01.331Z
131772023-12-10T18:55:01.331ZVerifying data now
131782023-12-10T18:55:06.777ZNo change after verify
131792023-12-10T18:55:06.777ZLoop: 53 Downstairs dump after verify (and repair):
131802023-12-10T18:55:06.800ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
131812023-12-10T18:55:06.800Z 0 000-019 100 100 100 179 179 179 F F F
131822023-12-10T18:55:06.800Z 1 020-039 100 100 100 179 179 179 F F F
131832023-12-10T18:55:06.803Z 2 040-059 107 107 107 188 188 188 F F F
131842023-12-10T18:55:06.803Z 3 060-079 107 107 107 188 188 188 F F F
131852023-12-10T18:55:06.803Z 4 080-099 106 106 106 185 185 185 F F F
131862023-12-10T18:55:06.804Z 5 100-119 103 103 103 182 182 182 F F F
131872023-12-10T18:55:06.804Z 6 120-139 102 102 102 181 181 181 F F F
131882023-12-10T18:55:06.804Z 7 140-159 92 92 92 165 165 165 F F F
131892023-12-10T18:55:06.804Z 8 160-179 107 107 107 188 188 188 F F F
131902023-12-10T18:55:06.804Z 9 180-199 107 107 107 188 188 188 F F F
131912023-12-10T18:55:06.804Z 10 200-219 107 107 107 188 188 188 F F F
131922023-12-10T18:55:06.804Z 11 220-239 105 105 105 184 184 184 F F F
131932023-12-10T18:55:06.804Z 12 240-259 106 106 106 185 185 185 F F F
131942023-12-10T18:55:06.804Z 13 260-279 106 106 106 185 185 185 F F F
131952023-12-10T18:55:06.804Z 14 280-299 98 98 98 176 176 176 F F F
131962023-12-10T18:55:06.804Z 15 300-319 106 106 106 187 187 187 F F F
131972023-12-10T18:55:06.804Z 16 320-339 105 105 105 184 184 184 F F F
131982023-12-10T18:55:06.804Z 17 340-359 103 103 103 182 182 182 F F F
131992023-12-10T18:55:06.804Z 18 360-379 107 107 107 188 188 188 F F F
132002023-12-10T18:55:06.804Z 19 380-399 107 107 107 188 188 188 F F F
132012023-12-10T18:55:06.804Z 20 400-419 105 105 105 184 184 184 F F F
132022023-12-10T18:55:06.804Z 21 420-439 105 105 105 184 184 184 F F F
132032023-12-10T18:55:06.805Z 22 440-459 96 96 96 174 174 174 F F F
132042023-12-10T18:55:06.805Z 23 460-479 104 104 104 183 183 183 F F F
132052023-12-10T18:55:06.805Z 24 480-499 104 104 104 183 183 183 F F F
132062023-12-10T18:55:06.805Z 25 500-519 106 106 106 186 186 186 F F F
132072023-12-10T18:55:06.805Z 26 520-539 106 106 106 186 186 186 F F F
132082023-12-10T18:55:06.805Z 27 540-559 107 107 107 188 188 188 F F F
132092023-12-10T18:55:06.806Z 28 560-579 106 106 106 185 185 185 F F F
132102023-12-10T18:55:06.806Z 29 580-599 103 103 103 182 182 182 F F F
132112023-12-10T18:55:06.806ZMax gen: 107, Max flush: 188
132122023-12-10T18:55:06.809Z
132132023-12-10T18:55:06.809ZBegin loop 54
132142023-12-10T18:55:06.809ZDownstairs to restart: 2
132152023-12-10T18:55:06.837Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.843492083Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594}
132162023-12-10T18:55:06.838Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.843915373Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594}
132172023-12-10T18:55:06.845Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.843952328Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594}
132182023-12-10T18:55:06.845Z{"msg":"Crucible b98c9e8e-5519-42dd-ac92-0d54a822672c has session id: 7ffa836c-f69e-4c6f-8f63-a79dca8018fb","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.844032104Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132192023-12-10T18:55:06.846Z{"msg":"Upstairs opts: Upstairs UUID: b98c9e8e-5519-42dd-ac92-0d54a822672c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.844090488Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132202023-12-10T18:55:06.846Z{"msg":"Crucible stats registered with UUID: b98c9e8e-5519-42dd-ac92-0d54a822672c","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.84416761Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132212023-12-10T18:55:06.846ZCrucible runtime is spawned
132222023-12-10T18:55:06.846ZThe guest has requested activation with gen:108
132232023-12-10T18:55:06.846Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.844511304Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb","task":"up_listen"}
132242023-12-10T18:55:06.846Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.844588685Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"looper":"0","session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132252023-12-10T18:55:06.846Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.844621453Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132262023-12-10T18:55:06.846Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.844650932Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132272023-12-10T18:55:06.846Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.844904805Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"looper":"1","session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132282023-12-10T18:55:06.846Z{"msg":"[0] b98c9e8e-5519-42dd-ac92-0d54a822672c looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.844974213Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"looper":"0","session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132292023-12-10T18:55:06.846Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.845043457Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"looper":"2","session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132302023-12-10T18:55:06.846Z{"msg":"[1] b98c9e8e-5519-42dd-ac92-0d54a822672c looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.845191869Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"looper":"1","session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132312023-12-10T18:55:06.846Z{"msg":"[2] b98c9e8e-5519-42dd-ac92-0d54a822672c looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.845279992Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"looper":"2","session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132322023-12-10T18:55:06.846Z{"msg":"b98c9e8e-5519-42dd-ac92-0d54a822672c active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.845724374Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132332023-12-10T18:55:06.846Z{"msg":"Set desired generation to :108","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.845785961Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132342023-12-10T18:55:06.846Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.845900246Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132352023-12-10T18:55:06.847Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.84625098Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132362023-12-10T18:55:06.847Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.846427636Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132372023-12-10T18:55:06.847Z{"msg":"[0] b98c9e8e-5519-42dd-ac92-0d54a822672c (7ffa836c-f69e-4c6f-8f63-a79dca8018fb) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.846706914Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132382023-12-10T18:55:06.847Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.846800414Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132392023-12-10T18:55:06.847Z{"msg":"[0] client is_active_req TRUE, promote! session 7ffa836c-f69e-4c6f-8f63-a79dca8018fb","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.846826669Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132402023-12-10T18:55:06.847Z{"msg":"[1] b98c9e8e-5519-42dd-ac92-0d54a822672c (7ffa836c-f69e-4c6f-8f63-a79dca8018fb) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.846850371Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132412023-12-10T18:55:06.847Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.846912436Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132422023-12-10T18:55:06.847Z{"msg":"[1] client is_active_req TRUE, promote! session 7ffa836c-f69e-4c6f-8f63-a79dca8018fb","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.846938326Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132432023-12-10T18:55:06.847Z{"msg":"[2] b98c9e8e-5519-42dd-ac92-0d54a822672c (7ffa836c-f69e-4c6f-8f63-a79dca8018fb) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.847334255Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132442023-12-10T18:55:06.847Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.847390583Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132452023-12-10T18:55:06.847Z{"msg":"[2] client is_active_req TRUE, promote! session 7ffa836c-f69e-4c6f-8f63-a79dca8018fb","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.847416651Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132462023-12-10T18:55:06.847Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.847438431Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132472023-12-10T18:55:06.847Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.847482459Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132482023-12-10T18:55:06.847Z{"msg":"b98c9e8e-5519-42dd-ac92-0d54a822672c WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.847511007Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132492023-12-10T18:55:06.847Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.847769908Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132502023-12-10T18:55:06.847Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.847857913Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132512023-12-10T18:55:06.848Z{"msg":"b98c9e8e-5519-42dd-ac92-0d54a822672c WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.847941929Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132522023-12-10T18:55:06.848Z{"msg":"[0] b98c9e8e-5519-42dd-ac92-0d54a822672c (7ffa836c-f69e-4c6f-8f63-a79dca8018fb) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.848193377Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132532023-12-10T18:55:06.848Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.848261682Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132542023-12-10T18:55:06.848Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:06.848348465Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132552023-12-10T18:55:06.848Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.848404787Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132562023-12-10T18:55:06.848Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.848444443Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132572023-12-10T18:55:06.848Z{"msg":"b98c9e8e-5519-42dd-ac92-0d54a822672c WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.848482667Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132582023-12-10T18:55:06.848Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.848519899Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132592023-12-10T18:55:06.848Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.848557594Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132602023-12-10T18:55:06.853Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.848596121Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132612023-12-10T18:55:06.853Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.848662385Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132622023-12-10T18:55:06.853Z{"msg":"b98c9e8e-5519-42dd-ac92-0d54a822672c WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.848704344Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132632023-12-10T18:55:06.853Z{"msg":"[1] b98c9e8e-5519-42dd-ac92-0d54a822672c (7ffa836c-f69e-4c6f-8f63-a79dca8018fb) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.848762273Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132642023-12-10T18:55:06.853Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.848801991Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132652023-12-10T18:55:06.853Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:06.848855946Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132662023-12-10T18:55:06.853Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.848894Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132672023-12-10T18:55:06.853Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.848951261Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132682023-12-10T18:55:06.853Z{"msg":"b98c9e8e-5519-42dd-ac92-0d54a822672c WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.848991702Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132692023-12-10T18:55:06.853Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.849050271Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132702023-12-10T18:55:06.854Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.849088793Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132712023-12-10T18:55:06.854Z{"msg":"[2] b98c9e8e-5519-42dd-ac92-0d54a822672c (7ffa836c-f69e-4c6f-8f63-a79dca8018fb) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.849336327Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132722023-12-10T18:55:06.854Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.849414004Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132732023-12-10T18:55:06.854Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:06.849456675Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132742023-12-10T18:55:06.854Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.849526508Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132752023-12-10T18:55:06.854Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.849566884Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132762023-12-10T18:55:06.854Z{"msg":"b98c9e8e-5519-42dd-ac92-0d54a822672c WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.84963799Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132772023-12-10T18:55:06.854Z{"msg":"[0]R flush_numbers[0..12]: [179, 179, 188, 188, 185, 182, 181, 165, 188, 188, 188, 184]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.8497095Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132782023-12-10T18:55:06.854Z{"msg":"[0]R generation[0..12]: [100, 100, 107, 107, 106, 103, 102, 92, 107, 107, 107, 105]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.849779824Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132792023-12-10T18:55:06.854Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.849823153Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132802023-12-10T18:55:06.855Z{"msg":"[1]R flush_numbers[0..12]: [179, 179, 188, 188, 185, 182, 181, 165, 188, 188, 188, 184]"The guest has finished waiting for activation with:108
132812023-12-10T18:55:06.855Z,"v":0,"name":"crucible","levelWait for a query_work_queue command to finish before sending IO
132822023-12-10T18:55:06.855Z":30,"time":"2023-12-10T18:55:06.849934019Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132832023-12-10T18:55:06.855Z{"msg":"[1]R generation[0..12]: [100, 100, 107, 107, 106, 103, 102, 92, 107, 107, 107, 105]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.84999129Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132842023-12-10T18:55:06.855Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.850087305Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132852023-12-10T18:55:06.855Z{"msg":"[2]R flush_numbers[0..12]: [179, 179, 188, 188, 185, 182, 181, 165, 188, 188, 188, 184]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.850131637Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132862023-12-10T18:55:06.855Z{"msg":"[2]R generation[0..12]: [100, 100, 107, 107, 106, 103, 102, 92, 107, 107, 107, 105]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.850227131Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132872023-12-10T18:55:06.855Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.850272125Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132882023-12-10T18:55:06.855Z{"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.850307707Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132892023-12-10T18:55:06.855Z{"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.850373542Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132902023-12-10T18:55:06.855Z{"msg":"Next flush: 189","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.850399369Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132912023-12-10T18:55:06.855Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.850470088Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132922023-12-10T18:55:06.855Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.850542732Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132932023-12-10T18:55:06.855Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.850615632Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132942023-12-10T18:55:06.855Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.850686866Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132952023-12-10T18:55:06.855Z{"msg":"b98c9e8e-5519-42dd-ac92-0d54a822672c is now active with session: 7ffa836c-f69e-4c6f-8f63-a79dca8018fb","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.850763569Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132962023-12-10T18:55:06.855Z{"msg":"b98c9e8e-5519-42dd-ac92-0d54a822672c Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.850831758Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132972023-12-10T18:55:06.856Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.850899312Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132982023-12-10T18:55:06.856Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.850936821Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
132992023-12-10T18:55:06.856Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.850981469Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
133002023-12-10T18:55:06.856Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
133012023-12-10T18:55:06.856Z2023-12-10T18:55:06.851003678Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
133022023-12-10T18:55:06.856Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.851138394Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
133032023-12-10T18:55:06.857Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.851164668Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
133042023-12-10T18:55:06.857Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.851204662Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
133052023-12-10T18:55:06.857Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.851226911Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
133062023-12-10T18:55:06.857Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.851266275Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
133072023-12-10T18:55:06.857Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.851288009Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
133082023-12-10T18:55:06.857Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:06.851327067Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1594,"session_id":"7ffa836c-f69e-4c6f-8f63-a79dca8018fb"}
133092023-12-10T18:55:06.857ZLoading write count information from file "/tmp/test_repair_verify.data"
133102023-12-10T18:55:06.857ZRun Repair workload
133112023-12-10T18:55:06.857Z01/30 Write block 417 len 28672 data: 7 8 9 8 10 9 9
133122023-12-10T18:55:06.857Z02/30 Read block 553 len 4096
133132023-12-10T18:55:06.857Z03/30 Read block 369 len 16384
133142023-12-10T18:55:06.857Z04/30 Write block 573 len 20480 data: 8 9 10 11 11
133152023-12-10T18:55:06.858Z05/30 Write block 315 len 16384 data: 6 4 4 7
133162023-12-10T18:55:06.861Z06/30 Read block 247 len 20480
133172023-12-10T18:55:06.861Z07/30 Read block 420 len 24576
133182023-12-10T18:55:06.861Z08/30 Read block 316 len 12288
133192023-12-10T18:55:06.861Z09/30 Read block 449 len 28672
133202023-12-10T18:55:06.862Z10/30 Write block 92 len 16384 data: 8 10 10 10
133212023-12-10T18:55:06.864Z11/30 Read block 339 len 32768
133222023-12-10T18:55:06.867Z12/30 Read block 158 len 16384
133232023-12-10T18:55:06.867Z13/30 Read block 468 len 4096
133242023-12-10T18:55:06.870Z14/30 Flush
133252023-12-10T18:55:06.873Z15/30 Write block 310 len 32768 data: 4 4 4 4 7 7 5 5
133262023-12-10T18:55:06.873Z16/30 Read block 234 len 4096
133272023-12-10T18:55:06.876Z17/30 Read block 271 len 12288
133282023-12-10T18:55:06.876Z18/30 Write block 55 len 16384 data: 4 5 6 5
133292023-12-10T18:55:06.878Z19/30 Write block 139 len 32768 data: 5 4 4 4 4 5 5 7
133302023-12-10T18:55:06.878Z20/30 Read block 571 len 40960
133312023-12-10T18:55:06.881Z21/30 Flush
133322023-12-10T18:55:06.884Z22/30 Write block 30 len 12288 data: 7 8 11
133332023-12-10T18:55:06.887Z23/30 Read block 382 len 36864
133342023-12-10T18:55:06.889Z24/30 Flush
133352023-12-10T18:55:06.892Z25/30 Write block 526 len 8192 data: 8 7
133362023-12-10T18:55:06.892Z26/30 Read block 381 len 36864
133372023-12-10T18:55:06.895Z27/30 Write block 537 len 36864 data: 7 8 8 11 12 13 13 12 8
133382023-12-10T18:55:06.898Z28/30 Write block 173 len 28672 data: 10 9 7 7 8 6 4
133392023-12-10T18:55:06.898Z29/30 Write block 526 len 12288 data: 9 8 7
133402023-12-10T18:55:06.898Z30/30 Read block 529 len 32768
133412023-12-10T18:55:06.901Z----------------------------------------------------------------
133422023-12-10T18:55:06.901Z Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30
133432023-12-10T18:55:06.901ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
133442023-12-10T18:55:06.901Z 1 Acked 1000 Write 7 Done Done Sent false
133452023-12-10T18:55:06.902Z 2 Acked 1001 Read 1 Done Done Sent false
133462023-12-10T18:55:06.902Z 3 Acked 1002 Read 4 Done Done Sent false
133472023-12-10T18:55:06.902Z 4 Acked 1003 Write 5 Done Done Sent false
133482023-12-10T18:55:06.903Z 5 Acked 1004 Write 4 Done Done Sent false
133492023-12-10T18:55:06.903Z 6 Acked 1005 Read 5 Done Done Sent false
133502023-12-10T18:55:06.903Z 7 Acked 1006 Read 6 Done Done Sent false
133512023-12-10T18:55:06.903Z 8 Acked 1007 Read 3 Done Done Sent false
133522023-12-10T18:55:06.903Z 9 Acked 1008 Read 7 Done Done Sent false
133532023-12-10T18:55:06.903Z 10 Acked 1009 Write 4 Done Done Sent false
133542023-12-10T18:55:06.903Z 11 Acked 1010 Read 8 Done Done Sent false
133552023-12-10T18:55:06.903Z 12 Acked 1011 Read 4 Done Done Sent false
133562023-12-10T18:55:06.903Z 13 Acked 1012 Read 1 Done Done Sent false
133572023-12-10T18:55:06.903Z 14 Acked 1013 Flush 0 Done Done Sent false
133582023-12-10T18:55:06.903Z 15 Acked 1014 Write 8 Done Done Sent false
133592023-12-10T18:55:06.903Z 16 Acked 1015 Read 1 Done Done Sent false
133602023-12-10T18:55:06.903Z 17 Acked 1016 Read 3 Done Done Sent false
133612023-12-10T18:55:06.903Z 18 Acked 1017 Write 4 Done Done Sent false
133622023-12-10T18:55:06.903Z 19 Acked 1018 Write 8 Done Done Sent false
133632023-12-10T18:55:06.903Z 20 Acked 1019 Read 10 Done Done Sent false
133642023-12-10T18:55:06.903Z 21 Acked 1020 Flush 0 Done Done Sent false
133652023-12-10T18:55:06.903Z 22 Acked 1021 Write 3 Done Done Sent false
133662023-12-10T18:55:06.904Z 23 Acked 1022 Read 9 Done Done Sent false
133672023-12-10T18:55:06.904Z 24 Acked 1023 Flush 0 Done Done Sent false
133682023-12-10T18:55:06.904Z 25 Acked 1024 Write 2 Done Done Sent false
133692023-12-10T18:55:06.904Z 26 Acked 1025 Read 9 Done Done Sent false
133702023-12-10T18:55:06.904Z 27 Acked 1026 Write 9 Done Done Sent false
133712023-12-10T18:55:06.904Z 28 Acked 1027 Write 7 Done Done Sent false
133722023-12-10T18:55:06.904Z 29 Acked 1028 Write 3 Done Done Sent false
133732023-12-10T18:55:06.904Z 30 Acked 1029 Read 8 Done Done Sent false
133742023-12-10T18:55:06.904Z STATES DS:0 DS:1 DS:2 TOTAL
133752023-12-10T18:55:06.904Z New 0 0 0 0
133762023-12-10T18:55:06.904Z Sent 0 0 30 30
133772023-12-10T18:55:06.904Z Done 30 30 0 60
133782023-12-10T18:55:06.904Z Skipped 0 0 0 0
133792023-12-10T18:55:06.904Z Error 0 0 0 0
133802023-12-10T18:55:06.904ZLast Flush: 1023 1023 0
133812023-12-10T18:55:06.904ZDownstairs last five completed:
133822023-12-10T18:55:06.904ZUpstairs last five completed: 30 29 28 27 26 25
133832023-12-10T18:55:06.905ZWrote out file "/tmp/test_repair_verify.data"
133842023-12-10T18:55:06.912Z
133852023-12-10T18:55:06.915ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
133862023-12-10T18:55:06.937ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
133872023-12-10T18:55:06.937Z 0 000-019 100 100 100 179 179 179 F F F
133882023-12-10T18:55:06.937Z 1 020-039 108 108 100 191 191 179 F F F <---
133892023-12-10T18:55:06.937Z 2 040-059 108 108 107 190 190 188 F F F <---
133902023-12-10T18:55:06.937Z 3 060-079 107 107 107 188 188 188 F F F
133912023-12-10T18:55:06.937Z 4 080-099 108 108 106 189 189 185 F F F <---
133922023-12-10T18:55:06.938Z 5 100-119 103 103 103 182 182 182 F F F
133932023-12-10T18:55:06.939Z 6 120-139 108 108 102 190 190 181 F F F <---
133942023-12-10T18:55:06.939Z 7 140-159 108 108 92 190 190 165 F F F <---
133952023-12-10T18:55:06.945Z 8 160-179 107 107 107 188 188 188 T T F <---
133962023-12-10T18:55:06.945Z 9 180-199 107 107 107 188 188 188 F F F
133972023-12-10T18:55:06.945Z 10 200-219 107 107 107 188 188 188 F F F
133982023-12-10T18:55:06.945Z 11 220-239 105 105 105 184 184 184 F F F
133992023-12-10T18:55:06.945Z 12 240-259 106 106 106 185 185 185 F F F
134002023-12-10T18:55:06.945Z 13 260-279 106 106 106 185 185 185 F F F
134012023-12-10T18:55:06.945Z 14 280-299 98 98 98 176 176 176 F F F
134022023-12-10T18:55:06.945Z 15 300-319 108 108 106 190 190 187 F F F <---
134032023-12-10T18:55:06.945Z 16 320-339 105 105 105 184 184 184 F F F
134042023-12-10T18:55:06.945Z 17 340-359 103 103 103 182 182 182 F F F
134052023-12-10T18:55:06.945Z 18 360-379 107 107 107 188 188 188 F F F
134062023-12-10T18:55:06.946Z 19 380-399 107 107 107 188 188 188 F F F
134072023-12-10T18:55:06.946Z 20 400-419 108 108 105 189 189 184 F F F <---
134082023-12-10T18:55:06.946Z 21 420-439 108 108 105 189 189 184 F F F <---
134092023-12-10T18:55:06.946Z 22 440-459 96 96 96 174 174 174 F F F
134102023-12-10T18:55:06.946Z 23 460-479 104 104 104 183 183 183 F F F
134112023-12-10T18:55:06.946Z 24 480-499 104 104 104 183 183 183 F F F
134122023-12-10T18:55:06.946Z 25 500-519 106 106 106 186 186 186 F F F
134132023-12-10T18:55:06.946Z 26 520-539 106 106 106 186 186 186 T T F <---
134142023-12-10T18:55:06.946Z 27 540-559 107 107 107 188 188 188 T T F <---
134152023-12-10T18:55:06.946Z 28 560-579 108 108 106 189 189 185 F F F <---
134162023-12-10T18:55:06.947Z 29 580-599 103 103 103 182 182 182 F F F
134172023-12-10T18:55:06.947ZMax gen: 108, Max flush: 191
134182023-12-10T18:55:06.947ZError: Difference in extent metadata found!
134192023-12-10T18:55:06.947ZA Difference in extent metadata is expected here
134202023-12-10T18:55:06.947ZOn loop 54
134212023-12-10T18:55:06.947Z
134222023-12-10T18:55:06.950ZVerifying data now
134232023-12-10T18:55:12.140ZNo change after verify
134242023-12-10T18:55:12.140ZLoop: 54 Downstairs dump after verify (and repair):
134252023-12-10T18:55:12.167ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
134262023-12-10T18:55:12.167Z 0 000-019 100 100 100 179 179 179 F F F
134272023-12-10T18:55:12.814Z 1 020-039 108 108 108 191 191 191 F F F
134282023-12-10T18:55:12.815Z 2 040-059 108 108 108 190 190 190 F F F
134292023-12-10T18:55:12.815Z 3 060-079 107 107 107 188 188 188 F F F
134302023-12-10T18:55:12.815Z 4 080-099 108 108 108 189 189 189 F F F
134312023-12-10T18:55:12.815Z 5 100-119 103 103 103 182 182 182 F F F
134322023-12-10T18:55:12.815Z 6 120-139 108 108 108 190 190 190 F F F
134332023-12-10T18:55:12.815Z 7 140-159 108 108 108 190 190 190 F F F
134342023-12-10T18:55:12.815Z 8 160-179 109 109 109 192 192 192 F F F
134352023-12-10T18:55:12.815Z 9 180-199 107 107 107 188 188 188 F F F
134362023-12-10T18:55:12.815Z 10 200-219 107 107 107 188 188 188 F F F
134372023-12-10T18:55:12.815Z 11 220-239 105 105 105 184 184 184 F F F
134382023-12-10T18:55:12.815Z 12 240-259 106 106 106 185 185 185 F F F
134392023-12-10T18:55:12.815Z 13 260-279 106 106 106 185 185 185 F F F
134402023-12-10T18:55:12.816Z 14 280-299 98 98 98 176 176 176 F F F
134412023-12-10T18:55:12.816Z 15 300-319 108 108 108 190 190 190 F F F
134422023-12-10T18:55:12.816Z 16 320-339 105 105 105 184 184 184 F F F
134432023-12-10T18:55:12.816Z 17 340-359 103 103 103 182 182 182 F F F
134442023-12-10T18:55:12.816Z 18 360-379 107 107 107 188 188 188 F F F
134452023-12-10T18:55:12.816Z 19 380-399 107 107 107 188 188 188 F F F
134462023-12-10T18:55:12.816Z 20 400-419 108 108 108 189 189 189 F F F
134472023-12-10T18:55:12.816Z 21 420-439 108 108 108 189 189 189 F F F
134482023-12-10T18:55:12.816Z 22 440-459 96 96 96 174 174 174 F F F
134492023-12-10T18:55:12.816Z 23 460-479 104 104 104 183 183 183 F F F
134502023-12-10T18:55:12.816Z 24 480-499 104 104 104 183 183 183 F F F
134512023-12-10T18:55:12.816Z 25 500-519 106 106 106 186 186 186 F F F
134522023-12-10T18:55:12.817Z 26 520-539 109 109 109 192 192 192 F F F
134532023-12-10T18:55:12.817Z 27 540-559 109 109 109 192 192 192 F F F
134542023-12-10T18:55:12.817Z 28 560-579 108 108 108 189 189 189 F F F
134552023-12-10T18:55:12.817Z 29 580-599 103 103 103 182 182 182 F F F
134562023-12-10T18:55:12.817ZMax gen: 109, Max flush: 192
134572023-12-10T18:55:12.817Z
134582023-12-10T18:55:12.817ZBegin loop 55
134592023-12-10T18:55:12.817ZDownstairs to restart: 1
134602023-12-10T18:55:12.826Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.20107944Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602}
134612023-12-10T18:55:12.826Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.201493752Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602}
134622023-12-10T18:55:12.826Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.201546137Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602}
134632023-12-10T18:55:12.826Z{"msg":"Crucible 4fdbc61b-75b7-48d4-96fe-d67e09faca86 has session id: e4062287-7dcd-497f-ad98-4b900801f1bc","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.201571088Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134642023-12-10T18:55:12.826Z{"msg":"Upstairs opts: Upstairs UUID: 4fdbc61b-75b7-48d4-96fe-d67e09faca86, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.201612276Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134652023-12-10T18:55:12.827Z{"msg":"Crucible stats registered with UUID: 4fdbc61b-75b7-48d4-96fe-d67e09faca86","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.201686535Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134662023-12-10T18:55:12.827ZCrucible runtime is spawned
134672023-12-10T18:55:12.827Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30The guest has requested activation with gen:110
134682023-12-10T18:55:12.827Z,"time":"2023-12-10T18:55:12.201927998Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc","task":"up_listen"}
134692023-12-10T18:55:12.827Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.202106744Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"looper":"0","session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134702023-12-10T18:55:12.827Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.202144504Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134712023-12-10T18:55:12.827Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.202204612Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134722023-12-10T18:55:12.827Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.202230814Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"looper":"2","session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134732023-12-10T18:55:12.827Z{"msg":"[0] 4fdbc61b-75b7-48d4-96fe-d67e09faca86 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.202354688Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"looper":"0","session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134742023-12-10T18:55:12.827Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.202405514Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"looper":"1","session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134752023-12-10T18:55:12.827Z{"msg":"[1] 4fdbc61b-75b7-48d4-96fe-d67e09faca86 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.202481213Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"looper":"1","session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134762023-12-10T18:55:12.827Z{"msg":"[2] 4fdbc61b-75b7-48d4-96fe-d67e09faca86 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.202564289Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"looper":"2","session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134772023-12-10T18:55:12.827Z{"msg":"4fdbc61b-75b7-48d4-96fe-d67e09faca86 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.202890214Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134782023-12-10T18:55:12.827Z{"msg":"Set desired generation to :110","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.202944209Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134792023-12-10T18:55:12.828Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.203114424Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134802023-12-10T18:55:12.828Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.203310448Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134812023-12-10T18:55:12.828Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.203458103Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134822023-12-10T18:55:12.828Z{"msg":"[0] 4fdbc61b-75b7-48d4-96fe-d67e09faca86 (e4062287-7dcd-497f-ad98-4b900801f1bc) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.203927578Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134832023-12-10T18:55:12.828Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.204016353Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134842023-12-10T18:55:12.828Z{"msg":"[2] 4fdbc61b-75b7-48d4-96fe-d67e09faca86 (e4062287-7dcd-497f-ad98-4b900801f1bc) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.204060842Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134852023-12-10T18:55:12.828Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.204138351Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134862023-12-10T18:55:12.828Z{"msg":"[0] client is_active_req TRUE, promote! session e4062287-7dcd-497f-ad98-4b900801f1bc","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.204182225Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134872023-12-10T18:55:12.828Z{"msg":"[2] client is_active_req TRUE, promote! session e4062287-7dcd-497f-ad98-4b900801f1bc","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.204256751Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134882023-12-10T18:55:12.829Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.205018623Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134892023-12-10T18:55:12.829Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.205142357Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134902023-12-10T18:55:12.829Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.205212616Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134912023-12-10T18:55:12.829Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.205311388Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134922023-12-10T18:55:12.829Z{"msg":"4fdbc61b-75b7-48d4-96fe-d67e09faca86 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.205341476Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134932023-12-10T18:55:12.829Z{"msg":"4fdbc61b-75b7-48d4-96fe-d67e09faca86 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.205364918Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134942023-12-10T18:55:12.829Z{"msg":"[1] 4fdbc61b-75b7-48d4-96fe-d67e09faca86 (e4062287-7dcd-497f-ad98-4b900801f1bc) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.205677373Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134952023-12-10T18:55:12.829Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.205764473Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134962023-12-10T18:55:12.829Z{"msg":"[1] client is_active_req TRUE, promote! session e4062287-7dcd-497f-ad98-4b900801f1bc","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.205846158Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134972023-12-10T18:55:12.830Z{"msg":"[2] 4fdbc61b-75b7-48d4-96fe-d67e09faca86 (e4062287-7dcd-497f-ad98-4b900801f1bc) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.205965829Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134982023-12-10T18:55:12.831Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.206031815Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
134992023-12-10T18:55:12.831Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:12.20607249Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135002023-12-10T18:55:12.831Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.206167491Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135012023-12-10T18:55:12.831Z{"msg":"[0] 4fdbc61b-75b7-48d4-96fe-d67e09faca86 (e4062287-7dcd-497f-ad98-4b900801f1bc) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.206212897Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135022023-12-10T18:55:12.831Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.206276828Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135032023-12-10T18:55:12.831Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:12.206316388Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135042023-12-10T18:55:12.831Z{"msg":"4fdbc61b-75b7-48d4-96fe-d67e09faca86 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.206381333Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135052023-12-10T18:55:12.831Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.206419458Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135062023-12-10T18:55:12.831Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.206454183Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135072023-12-10T18:55:12.831Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.206521702Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135082023-12-10T18:55:12.831Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.206552692Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135092023-12-10T18:55:12.831Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.206574981Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135102023-12-10T18:55:12.831Z{"msg":"4fdbc61b-75b7-48d4-96fe-d67e09faca86 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.206596033Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135112023-12-10T18:55:12.831Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.206638216Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135122023-12-10T18:55:12.831Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.206660972Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135132023-12-10T18:55:12.831Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.206682704Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135142023-12-10T18:55:12.831Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.206709754Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135152023-12-10T18:55:12.832Z{"msg":"4fdbc61b-75b7-48d4-96fe-d67e09faca86 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.20673128Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135162023-12-10T18:55:12.832Z{"msg":"[1] 4fdbc61b-75b7-48d4-96fe-d67e09faca86 (e4062287-7dcd-497f-ad98-4b900801f1bc) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.207232225Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135172023-12-10T18:55:12.832Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.207298109Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135182023-12-10T18:55:12.832Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:12.207322322Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135192023-12-10T18:55:12.832Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.2073431Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135202023-12-10T18:55:12.832Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.207365134Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135212023-12-10T18:55:12.833Z{"msg":"4fdbc61b-75b7-48d4-96fe-d67e09faca86 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.207406912Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135222023-12-10T18:55:12.833Z{"msg":"[0]R flush_numbers[0..12]: [179, 191, 190, 188, 189, 182, 190, 190, 192, 188, 188, 184]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.207431471Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135232023-12-10T18:55:12.833Z{"msg":"[0]R generation[0..12]: [100, 108, 108, 107, 108, 103, 108, 108, 109, 107, 107, 105]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.207473501Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135242023-12-10T18:55:12.833Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.207497665Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135252023-12-10T18:55:12.833Z{The guest has finished waiting for activation with:110
135262023-12-10T18:55:12.833Z"msg":"Wait for a query_work_queue command to finish before sending IO
135272023-12-10T18:55:12.833Z[1]R flush_numbers[0..12]: [179, 191, 190, 188, 189, 182, 190, 190, 192, 188, 188, 184]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.207565565Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135282023-12-10T18:55:12.833Z{"msg":"[1]R generation[0..12]: [100, 108, 108, 107, 108, 103, 108, 108, 109, 107, 107, 105]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.207604785Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135292023-12-10T18:55:12.834Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.207628997Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135302023-12-10T18:55:12.834Z{"msg":"[2]R flush_numbers[0..12]: [179, 191, 190, 188, 189, 182, 190, 190, 192, 188, 188, 184]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.207656521Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135312023-12-10T18:55:12.834Z{"msg":"[2]R generation[0..12]: [100, 108, 108, 107, 108, 103, 108, 108, 109, 107, 107, 105]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.207725509Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135322023-12-10T18:55:12.834Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.207770383Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135332023-12-10T18:55:12.834Z{"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.207845646Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135342023-12-10T18:55:12.834Z{"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.207872514Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135352023-12-10T18:55:12.834Z{"msg":"Next flush: 193","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.207892434Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135362023-12-10T18:55:12.834Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.207955055Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135372023-12-10T18:55:12.834Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.207995827Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135382023-12-10T18:55:12.835Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.208069168Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135392023-12-10T18:55:12.835Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.208140747Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135402023-12-10T18:55:12.835Z{"msg":"4fdbc61b-75b7-48d4-96fe-d67e09faca86 is now active with session: e4062287-7dcd-497f-ad98-4b900801f1bc","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.208213556Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135412023-12-10T18:55:12.835Z{"msg":"4fdbc61b-75b7-48d4-96fe-d67e09faca86 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.208289447Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135422023-12-10T18:55:12.835Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.208375068Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135432023-12-10T18:55:12.835Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.208442707Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135442023-12-10T18:55:12.835Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.208512537Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135452023-12-10T18:55:12.835Z{"msg":"[0] received reconcile message","v":0Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
135462023-12-10T18:55:12.835Z,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.20872Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135472023-12-10T18:55:12.835Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.208778159Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135482023-12-10T18:55:12.835Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.208802433Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135492023-12-10T18:55:12.836Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.208843114Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135502023-12-10T18:55:12.836Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.208865401Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135512023-12-10T18:55:12.836Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.208905519Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135522023-12-10T18:55:12.836Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.208927697Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135532023-12-10T18:55:12.836Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:12.208967483Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1602,"session_id":"e4062287-7dcd-497f-ad98-4b900801f1bc"}
135542023-12-10T18:55:12.836ZLoading write count information from file "/tmp/test_repair_verify.data"
135552023-12-10T18:55:12.836ZRun Repair workload
135562023-12-10T18:55:12.836Z01/30 Write block 303 len 24576 data: 11 12 8 9 6 6
135572023-12-10T18:55:12.836Z02/30 Read block 276 len 28672
135582023-12-10T18:55:12.836Z03/30 Write block 156 len 28672 data: 6 9 7 8 8 8 11
135592023-12-10T18:55:12.837Z04/30 Read block 428 len 16384
135602023-12-10T18:55:12.842Z05/30 Read block 562 len 16384
135612023-12-10T18:55:12.842Z06/30 Flush
135622023-12-10T18:55:12.842Z07/30 Write block 285 len 36864 data: 12 12 12 11 11 8 10 9 8
135632023-12-10T18:55:12.842Z08/30 Write block 202 len 16384 data: 13 15 13 12
135642023-12-10T18:55:12.842Z09/30 Read block 267 len 28672
135652023-12-10T18:55:12.842Z10/30 Write block 496 len 12288 data: 6 6 8
135662023-12-10T18:55:12.842Z11/30 Read block 552 len 20480
135672023-12-10T18:55:12.842Z12/30 Flush
135682023-12-10T18:55:12.843Z13/30 Write block 571 len 20480 data: 10 8 9 10 11
135692023-12-10T18:55:12.843Z14/30 Write block 161 len 40960 data: 9 12 12 12 12 13 13 11 9 9
135702023-12-10T18:55:12.843Z15/30 Write block 113 len 12288 data: 8 6 8
135712023-12-10T18:55:12.843Z16/30 Read block 22 len 32768
135722023-12-10T18:55:12.843Z17/30 Write block 480 len 40960 data: 4 5 6 7 7 6 8 8 9 11
135732023-12-10T18:55:12.843Z18/30 Read block 407 len 28672
135742023-12-10T18:55:12.843Z19/30 Write block 227 len 24576 data: 12 11 11 15 14 13
135752023-12-10T18:55:12.843Z20/30 Write block 62 len 40960 data: 3 4 6 7 8 7 9 9 9 11
135762023-12-10T18:55:12.843Z21/30 Read block 506 len 24576
135772023-12-10T18:55:12.843Z22/30 Write block 257 len 28672 data: 5 5 2 5 6 8 10
135782023-12-10T18:55:12.843Z23/30 Write block 405 len 16384 data: 11 10 11 12
135792023-12-10T18:55:12.843Z24/30 Flush
135802023-12-10T18:55:12.843Z25/30 Write block 162 len 8192 data: 13 13
135812023-12-10T18:55:12.843Z26/30 Read block 249 len 32768
135822023-12-10T18:55:12.844Z27/30 Read block 437 len 40960
135832023-12-10T18:55:12.844Z28/30 Write block 482 len 40960 data: 7 8 8 7 9 9 10 12 10 11
135842023-12-10T18:55:12.844Z29/30 Write block 589 len 28672 data: 9 10 10 9 8 7 6
135852023-12-10T18:55:12.844Z30/30 Write block 293 len 28672 data: 9 6 4 6 6 7 10
135862023-12-10T18:55:12.844Z----------------------------------------------------------------
135872023-12-10T18:55:12.844Z Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:24
135882023-12-10T18:55:12.844ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
135892023-12-10T18:55:12.844Z 7 Acked 1006 Write 9 Done Sent Done false
135902023-12-10T18:55:12.844Z 8 Acked 1007 Write 4 Done Sent Done false
135912023-12-10T18:55:12.844Z 9 Acked 1008 Read 7 Done Sent Done false
135922023-12-10T18:55:12.845Z 10 Acked 1009 Write 3 Done Sent Done false
135932023-12-10T18:55:12.845Z 11 Acked 1010 Read 5 Done Sent Done false
135942023-12-10T18:55:12.845Z 12 Acked 1011 Flush 0 Done Sent Done false
135952023-12-10T18:55:12.845Z 13 Acked 1012 Write 5 Done Sent Done false
135962023-12-10T18:55:12.845Z 14 Acked 1013 Write 10 Done Sent Done false
135972023-12-10T18:55:12.845Z 15 Acked 1014 Write 3 Done Sent Done false
135982023-12-10T18:55:12.845Z 16 Acked 1015 Read 8 Done Sent Done false
135992023-12-10T18:55:12.845Z 17 Acked 1016 Write 10 Done Sent Done false
136002023-12-10T18:55:12.846Z 18 Acked 1017 Read 7 Done Sent Done false
136012023-12-10T18:55:12.846Z 19 Acked 1018 Write 6 Done Sent Done false
136022023-12-10T18:55:12.846Z 20 Acked 1019 Write 10 Done Sent Done false
136032023-12-10T18:55:12.846Z 21 Acked 1020 Read 6 Done Sent Done false
136042023-12-10T18:55:12.846Z 22 Acked 1021 Write 7 Done Sent Done false
136052023-12-10T18:55:12.846Z 23 Acked 1022 Write 4 Done Sent Done false
136062023-12-10T18:55:12.846Z 24 Acked 1023 Flush 0 Done Sent Done false
136072023-12-10T18:55:12.846Z 25 Acked 1024 Write 2 Done Sent Done false
136082023-12-10T18:55:12.846Z 26 Acked 1025 Read 8 Done Sent Done false
136092023-12-10T18:55:12.846Z 27 Acked 1026 Read 10 Done Sent Done false
136102023-12-10T18:55:12.846Z 28 Acked 1027 Write 10 Done Sent Done false
136112023-12-10T18:55:12.846Z 29 Acked 1028 Write 7 Done Sent Done false
136122023-12-10T18:55:12.847Z 30 Acked 1029 Write 7 Sent Sent Sent false
136132023-12-10T18:55:12.847Z STATES DS:0 DS:1 DS:2 TOTAL
136142023-12-10T18:55:12.847Z New 0 0 0 0
136152023-12-10T18:55:12.849Z Sent 1 24 1 26
136162023-12-10T18:55:12.849Z Done 23 0 23 46
136172023-12-10T18:55:12.849Z Skipped 0 0 0 0
136182023-12-10T18:55:12.849Z Error 0 0 0 0
136192023-12-10T18:55:12.849ZLast Flush: 1023 1005 1023
136202023-12-10T18:55:12.849ZDownstairs last five completed: 1005 1004 1003 1002 1001 1000
136212023-12-10T18:55:12.849ZUpstairs last five completed: 30 29 28 27 26 25
136222023-12-10T18:55:12.849ZWrote out file "/tmp/test_repair_verify.data"
136232023-12-10T18:55:12.850Z
136242023-12-10T18:55:12.850ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
136252023-12-10T18:55:12.850ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
136262023-12-10T18:55:12.850Z 0 000-019 100 100 100 179 179 179 F F F
136272023-12-10T18:55:12.850Z 1 020-039 108 108 108 191 191 191 F F F
136282023-12-10T18:55:12.850Z 2 040-059 108 108 108 190 190 190 F F F
136292023-12-10T18:55:12.850Z 3 060-079 110 107 110 195 188 195 F F F <---
136302023-12-10T18:55:12.850Z 4 080-099 108 108 108 189 189 189 F F F
136312023-12-10T18:55:12.850Z 5 100-119 110 103 110 195 182 195 F F F <---
136322023-12-10T18:55:12.850Z 6 120-139 108 108 108 190 190 190 F F F
136332023-12-10T18:55:12.850Z 7 140-159 110 110 110 193 193 193 F F F
136342023-12-10T18:55:12.850Z 8 160-179 110 110 110 195 193 195 T F T <---
136352023-12-10T18:55:12.850Z 9 180-199 107 107 107 188 188 188 F F F
136362023-12-10T18:55:12.850Z 10 200-219 110 107 110 194 188 194 F F F <---
136372023-12-10T18:55:12.850Z 11 220-239 110 105 110 195 184 195 F F F <---
136382023-12-10T18:55:12.850Z 12 240-259 110 106 110 195 185 195 F F F <---
136392023-12-10T18:55:12.851Z 13 260-279 110 106 110 195 185 195 F F F <---
136402023-12-10T18:55:12.851Z 14 280-299 110 98 110 194 176 194 T F T <---
136412023-12-10T18:55:12.851Z 15 300-319 110 110 110 193 193 193 F F F
136422023-12-10T18:55:12.851Z 16 320-339 105 105 105 184 184 184 F F F
136432023-12-10T18:55:12.851Z 17 340-359 103 103 103 182 182 182 F F F
136442023-12-10T18:55:12.851Z 18 360-379 107 107 107 188 188 188 F F F
136452023-12-10T18:55:12.851Z 19 380-399 107 107 107 188 188 188 F F F
136462023-12-10T18:55:12.851Z 20 400-419 110 108 110 195 189 195 F F F <---
136472023-12-10T18:55:12.851Z 21 420-439 108 108 108 189 189 189 F F F
136482023-12-10T18:55:12.852Z 22 440-459 96 96 96 174 174 174 F F F
136492023-12-10T18:55:12.852Z 23 460-479 104 104 104 183 183 183 F F F
136502023-12-10T18:55:12.852Z 24 480-499 110 104 110 195 183 195 T F T <---
136512023-12-10T18:55:12.852Z 25 500-519 106 106 106 186 186 186 F F F
136522023-12-10T18:55:12.852Z 26 520-539 109 109 109 192 192 192 F F F
136532023-12-10T18:55:12.852Z 27 540-559 109 109 109 192 192 192 F F F
136542023-12-10T18:55:12.852Z 28 560-579 110 108 110 195 189 195 F F F <---
136552023-12-10T18:55:12.852Z 29 580-599 103 103 103 182 182 182 T F T <---
136562023-12-10T18:55:12.852ZMax gen: 110, Max flush: 195
136572023-12-10T18:55:12.852ZError: Difference in extent metadata found!
136582023-12-10T18:55:12.852ZA Difference in extent metadata is expected here
136592023-12-10T18:55:12.852ZOn loop 55
136602023-12-10T18:55:12.856Z
136612023-12-10T18:55:12.856ZVerifying data now
136622023-12-10T18:55:17.483ZNo change after verify
136632023-12-10T18:55:17.483ZLoop: 55 Downstairs dump after verify (and repair):
136642023-12-10T18:55:17.511ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
136652023-12-10T18:55:17.511Z 0 000-019 100 100 100 179 179 179 F F F
136662023-12-10T18:55:17.538Z 1 020-039 108 108 108 191 191 191 F F F
136672023-12-10T18:55:17.538Z 2 040-059 108 108 108 190 190 190 F F F
136682023-12-10T18:55:17.538Z 3 060-079 110 110 110 195 195 195 F F F
136692023-12-10T18:55:17.538Z 4 080-099 108 108 108 189 189 189 F F F
136702023-12-10T18:55:17.538Z 5 100-119 110 110 110 195 195 195 F F F
136712023-12-10T18:55:17.538Z 6 120-139 108 108 108 190 190 190 F F F
136722023-12-10T18:55:17.538Z 7 140-159 110 110 110 193 193 193 F F F
136732023-12-10T18:55:17.538Z 8 160-179 111 111 111 196 196 196 F F F
136742023-12-10T18:55:17.538Z 9 180-199 107 107 107 188 188 188 F F F
136752023-12-10T18:55:17.538Z 10 200-219 110 110 110 194 194 194 F F F
136762023-12-10T18:55:17.539Z 11 220-239 110 110 110 195 195 195 F F F
136772023-12-10T18:55:17.539Z 12 240-259 110 110 110 195 195 195 F F F
136782023-12-10T18:55:17.539Z 13 260-279 110 110 110 195 195 195 F F F
136792023-12-10T18:55:17.539Z 14 280-299 111 111 111 196 196 196 F F F
136802023-12-10T18:55:17.539Z 15 300-319 110 110 110 193 193 193 F F F
136812023-12-10T18:55:17.539Z 16 320-339 105 105 105 184 184 184 F F F
136822023-12-10T18:55:17.539Z 17 340-359 103 103 103 182 182 182 F F F
136832023-12-10T18:55:17.539Z 18 360-379 107 107 107 188 188 188 F F F
136842023-12-10T18:55:17.539Z 19 380-399 107 107 107 188 188 188 F F F
136852023-12-10T18:55:17.539Z 20 400-419 110 110 110 195 195 195 F F F
136862023-12-10T18:55:17.539Z 21 420-439 108 108 108 189 189 189 F F F
136872023-12-10T18:55:17.539Z 22 440-459 96 96 96 174 174 174 F F F
136882023-12-10T18:55:17.540Z 23 460-479 104 104 104 183 183 183 F F F
136892023-12-10T18:55:17.540Z 24 480-499 111 111 111 196 196 196 F F F
136902023-12-10T18:55:17.540Z 25 500-519 106 106 106 186 186 186 F F F
136912023-12-10T18:55:17.540Z 26 520-539 109 109 109 192 192 192 F F F
136922023-12-10T18:55:17.540Z 27 540-559 109 109 109 192 192 192 F F F
136932023-12-10T18:55:17.540Z 28 560-579 110 110 110 195 195 195 F F F
136942023-12-10T18:55:17.540Z 29 580-599 111 111 111 196 196 196 F F F
136952023-12-10T18:55:17.540ZMax gen: 111, Max flush: 196
136962023-12-10T18:55:17.540Z
136972023-12-10T18:55:17.540ZBegin loop 56
136982023-12-10T18:55:17.540ZDownstairs to restart: 1
136992023-12-10T18:55:17.569Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.545865169Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610}
137002023-12-10T18:55:17.570Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.546252725Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610}
137012023-12-10T18:55:17.570Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.546326799Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610}
137022023-12-10T18:55:17.570ZCrucible runtime is spawned
137032023-12-10T18:55:17.571Z{"msg":"Crucible a6c668fe-924f-48b5-ae5e-852df8f5195b has session id: 9e72e929-b76b-40bb-9dda-32f58cc6cc6a","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.546372228Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137042023-12-10T18:55:17.571ZThe guest has requested activation with gen:112
137052023-12-10T18:55:17.571Z{"msg":"Upstairs opts: Upstairs UUID: a6c668fe-924f-48b5-ae5e-852df8f5195b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.546469567Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137062023-12-10T18:55:17.571Z{"msg":"Crucible stats registered with UUID: a6c668fe-924f-48b5-ae5e-852df8f5195b","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.546550666Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137072023-12-10T18:55:17.571Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.546708712Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"looper":"0","session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137082023-12-10T18:55:17.571Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.546801204Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a","task":"up_listen"}
137092023-12-10T18:55:17.571Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.546845667Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"looper":"1","session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137102023-12-10T18:55:17.571Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.54692591Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"looper":"2","session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137112023-12-10T18:55:17.571Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.546970143Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137122023-12-10T18:55:17.571Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.547036986Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137132023-12-10T18:55:17.571Z{"msg":"a6c668fe-924f-48b5-ae5e-852df8f5195b active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.547138032Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137142023-12-10T18:55:17.571Z{"msg":"Set desired generation to :112","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.547259883Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137152023-12-10T18:55:17.571Z{"msg":"[1] a6c668fe-924f-48b5-ae5e-852df8f5195b looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.547355166Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"looper":"1","session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137162023-12-10T18:55:17.571Z{"msg":"[2] a6c668fe-924f-48b5-ae5e-852df8f5195b looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.547399426Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"looper":"2","session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137172023-12-10T18:55:17.571Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.547469416Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137182023-12-10T18:55:17.571Z{"msg":"[0] a6c668fe-924f-48b5-ae5e-852df8f5195b looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.547511747Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"looper":"0","session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137192023-12-10T18:55:17.571Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.547656277Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137202023-12-10T18:55:17.572Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.547754843Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137212023-12-10T18:55:17.573Z{"msg":"[2] a6c668fe-924f-48b5-ae5e-852df8f5195b (9e72e929-b76b-40bb-9dda-32f58cc6cc6a) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.548148511Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137222023-12-10T18:55:17.573Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.548279691Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137232023-12-10T18:55:17.573Z{"msg":"[2] client is_active_req TRUE, promote! session 9e72e929-b76b-40bb-9dda-32f58cc6cc6a","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.548442485Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137242023-12-10T18:55:17.573Z{"msg":"[0] a6c668fe-924f-48b5-ae5e-852df8f5195b (9e72e929-b76b-40bb-9dda-32f58cc6cc6a) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.548505479Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137252023-12-10T18:55:17.573Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.548530212Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137262023-12-10T18:55:17.573Z{"msg":"[0] client is_active_req TRUE, promote! session 9e72e929-b76b-40bb-9dda-32f58cc6cc6a","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.548556372Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137272023-12-10T18:55:17.573Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.548970441Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137282023-12-10T18:55:17.573Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.549046877Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137292023-12-10T18:55:17.573Z{"msg":"[1] a6c668fe-924f-48b5-ae5e-852df8f5195b (9e72e929-b76b-40bb-9dda-32f58cc6cc6a) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.549095518Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137302023-12-10T18:55:17.574Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.549167684Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137312023-12-10T18:55:17.574Z{"msg":"[1] client is_active_req TRUE, promote! session 9e72e929-b76b-40bb-9dda-32f58cc6cc6a","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.549212058Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137322023-12-10T18:55:17.574Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.549273646Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137332023-12-10T18:55:17.574Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.549304244Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137342023-12-10T18:55:17.574Z{"msg":"a6c668fe-924f-48b5-ae5e-852df8f5195b WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.549327858Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137352023-12-10T18:55:17.574Z{"msg":"a6c668fe-924f-48b5-ae5e-852df8f5195b WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.549369994Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137362023-12-10T18:55:17.574Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.549940767Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137372023-12-10T18:55:17.574Z{"msg":"[2] a6c668fe-924f-48b5-ae5e-852df8f5195b (9e72e929-b76b-40bb-9dda-32f58cc6cc6a) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.550023739Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137382023-12-10T18:55:17.574Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.550061778Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137392023-12-10T18:55:17.576Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.550129866Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137402023-12-10T18:55:17.576Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:17.550172649Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137412023-12-10T18:55:17.576Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.55024668Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137422023-12-10T18:55:17.576Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.550318638Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137432023-12-10T18:55:17.576Z{"msg":"[0] a6c668fe-924f-48b5-ae5e-852df8f5195b (9e72e929-b76b-40bb-9dda-32f58cc6cc6a) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.550398641Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137442023-12-10T18:55:17.576Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.550475706Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137452023-12-10T18:55:17.576Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:17.550502773Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137462023-12-10T18:55:17.576Z{"msg":"a6c668fe-924f-48b5-ae5e-852df8f5195b WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.550545662Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137472023-12-10T18:55:17.576Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.550577346Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137482023-12-10T18:55:17.576Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.550702722Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137492023-12-10T18:55:17.577Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.550754254Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137502023-12-10T18:55:17.577Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.550778188Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137512023-12-10T18:55:17.577Z{"msg":"a6c668fe-924f-48b5-ae5e-852df8f5195b WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.550827523Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137522023-12-10T18:55:17.577Z{"msg":"a6c668fe-924f-48b5-ae5e-852df8f5195b WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.550873415Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137532023-12-10T18:55:17.578Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.550905779Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137542023-12-10T18:55:17.578Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.550926166Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137552023-12-10T18:55:17.578Z{"msg":"[1] a6c668fe-924f-48b5-ae5e-852df8f5195b (9e72e929-b76b-40bb-9dda-32f58cc6cc6a) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.551211744Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137562023-12-10T18:55:17.578Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.55132402Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137572023-12-10T18:55:17.578Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:17.551392904Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137582023-12-10T18:55:17.578Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.551433728Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137592023-12-10T18:55:17.578Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.551488376Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137602023-12-10T18:55:17.578Z{"msg":"a6c668fe-924f-48b5-ae5e-852df8f5195b WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.551529331Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137612023-12-10T18:55:17.578Z{"msg":"[0]R flush_numbers[0..12]: [179, 191, 190, 195, 189, 195, 190, 193, 196, 188, 194, 195]"The guest has finished waiting for activation with:112
137622023-12-10T18:55:17.578Z,"v":0,"name":"Wait for a query_work_queue command to finish before sending IO
137632023-12-10T18:55:17.578Zcrucible","level":30,"time":"2023-12-10T18:55:17.551604662Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137642023-12-10T18:55:17.578Z{"msg":"[0]R generation[0..12]: [100, 108, 108, 110, 108, 110, 108, 110, 111, 107, 110, 110]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.551646599Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137652023-12-10T18:55:17.578Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.551685488Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137662023-12-10T18:55:17.578Z{"msg":"[1]R flush_numbers[0..12]: [179, 191, 190, 195, 189, 195, 190, 193, 196, 188, 194, 195]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.551720985Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137672023-12-10T18:55:17.578Z{"msg":"[1]R generation[0..12]: [100, 108, 108, 110, 108, 110, 108, 110, 111, 107, 110, 110]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.551807471Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137682023-12-10T18:55:17.579Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.5519216Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137692023-12-10T18:55:17.579Z{"msg":"[2]R flush_numbers[0..12]: [179, 191, 190, 195, 189, 195, 190, 193, 196, 188, 194, 195]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.551983709Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137702023-12-10T18:55:17.579Z{"msg":"[2]R generation[0..12]: [100, 108, 108, 110, 108, 110, 108, 110, 111, 107, 110, 110]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.552025382Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137712023-12-10T18:55:17.580Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.552095533Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137722023-12-10T18:55:17.580Z{"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.552152114Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137732023-12-10T18:55:17.580Z{"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.552174808Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137742023-12-10T18:55:17.580Z{"msg":"Next flush: 197","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.552196766Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137752023-12-10T18:55:17.580Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.552256766Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137762023-12-10T18:55:17.580Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.552282008Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137772023-12-10T18:55:17.580Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.552323088Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137782023-12-10T18:55:17.580Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.552352497Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137792023-12-10T18:55:17.580Z{"msg":"a6c668fe-924f-48b5-ae5e-852df8f5195b is now active with session: 9e72e929-b76b-40bb-9dda-32f58cc6cc6a","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
137802023-12-10T18:55:17.580Z,"time":"2023-12-10T18:55:17.552417615Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137812023-12-10T18:55:17.580Z{"msg":"a6c668fe-924f-48b5-ae5e-852df8f5195b Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.552527355Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137822023-12-10T18:55:17.580Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.552565582Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137832023-12-10T18:55:17.580Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.552600091Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137842023-12-10T18:55:17.581Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.552650023Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137852023-12-10T18:55:17.581Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.552673476Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137862023-12-10T18:55:17.581Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.552713177Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137872023-12-10T18:55:17.581Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.552734843Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137882023-12-10T18:55:17.581Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.552774169Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137892023-12-10T18:55:17.581Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.552795447Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137902023-12-10T18:55:17.581Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.55283483Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137912023-12-10T18:55:17.581Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.552856556Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137922023-12-10T18:55:17.581Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:17.552895433Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1610,"session_id":"9e72e929-b76b-40bb-9dda-32f58cc6cc6a"}
137932023-12-10T18:55:17.582ZLoading write count information from file "/tmp/test_repair_verify.data"
137942023-12-10T18:55:17.582ZRun Repair workload
137952023-12-10T18:55:17.583Z01/30 Write block 207 len 8192 data: 9 7
137962023-12-10T18:55:17.594Z02/30 Read block 542 len 24576
137972023-12-10T18:55:17.594Z03/30 Read block 141 len 28672
137982023-12-10T18:55:17.594Z04/30 Read block 151 len 40960
137992023-12-10T18:55:17.594Z05/30 Flush
138002023-12-10T18:55:17.594Z06/30 Write block 74 len 16384 data: 12 11 11 12
138012023-12-10T18:55:17.594Z07/30 Write block 110 len 28672 data: 9 8 8 9 7 9 10
138022023-12-10T18:55:17.594Z08/30 Write block 399 len 12288 data: 8 8 9
138032023-12-10T18:55:17.595Z09/30 Read block 259 len 32768
138042023-12-10T18:55:17.595Z10/30 Flush
138052023-12-10T18:55:17.595Z11/30 Write block 318 len 12288 data: 8 9 10
138062023-12-10T18:55:17.595Z12/30 Write block 154 len 12288 data: 10 6 7
138072023-12-10T18:55:17.595Z13/30 Write block 220 len 36864 data: 12 15 14 11 12 10 12 13 12
138082023-12-10T18:55:17.595Z14/30 Read block 63 len 28672
138092023-12-10T18:55:17.595Z15/30 Write block 376 len 8192 data: 14 13
138102023-12-10T18:55:17.596Z16/30 Read block 191 len 12288
138112023-12-10T18:55:17.597Z17/30 Read block 229 len 16384
138122023-12-10T18:55:17.597Z18/30 Flush
138132023-12-10T18:55:17.597Z19/30 Write block 348 len 20480 data: 9 12 10 11 12
138142023-12-10T18:55:17.597Z20/30 Write block 443 len 8192 data: 6 7
138152023-12-10T18:55:17.597Z21/30 Flush
138162023-12-10T18:55:17.598Z22/30 Write block 587 len 20480 data: 8 9 10 11 11
138172023-12-10T18:55:17.598Z23/30 Write block 267 len 24576 data: 10 11 11 10 9 9
138182023-12-10T18:55:17.598Z24/30 Read block 387 len 8192
138192023-12-10T18:55:17.598Z25/30 Read block 552 len 12288
138202023-12-10T18:55:17.598Z26/30 Write block 533 len 8192 data: 10 12
138212023-12-10T18:55:17.598Z27/30 Write block 62 len 40960 data: 4 5 7 8 9 8 10 10 10 12
138222023-12-10T18:55:18.017Z28/30 Write block 68 len 12288 data: 11 11 11
138232023-12-10T18:55:18.017Z29/30 Read block 149 len 36864
138242023-12-10T18:55:18.017Z30/30 Read block 191 len 8192
138252023-12-10T18:55:18.017Z----------------------------------------------------------------
138262023-12-10T18:55:18.018Z Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30
138272023-12-10T18:55:18.018ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
138282023-12-10T18:55:18.018Z 1 Acked 1000 Write 2 Done Done Done false
138292023-12-10T18:55:18.018Z 2 Acked 1001 Read 6 Done Done Done false
138302023-12-10T18:55:18.018Z 3 Acked 1002 Read 7 Done Done Done false
138312023-12-10T18:55:18.018Z 4 Acked 1003 Read 10 Done Done Done false
138322023-12-10T18:55:18.018Z 5 Acked 1004 Flush 0 Done Sent Done false
138332023-12-10T18:55:18.018Z 6 Acked 1005 Write 4 Done Sent Done false
138342023-12-10T18:55:18.018Z 7 Acked 1006 Write 7 Done Sent Done false
138352023-12-10T18:55:18.018Z 8 Acked 1007 Write 3 Done Sent Done false
138362023-12-10T18:55:18.019Z 9 Acked 1008 Read 8 Done Sent Done false
138372023-12-10T18:55:18.019Z 10 Acked 1009 Flush 0 Done Sent Done false
138382023-12-10T18:55:18.019Z 11 Acked 1010 Write 3 Done Sent Done false
138392023-12-10T18:55:18.019Z 12 Acked 1011 Write 3 Done Sent Done false
138402023-12-10T18:55:18.019Z 13 Acked 1012 Write 9 Done Sent Done false
138412023-12-10T18:55:18.019Z 14 Acked 1013 Read 7 Done Sent Done false
138422023-12-10T18:55:18.020Z 15 Acked 1014 Write 2 Done Sent Done false
138432023-12-10T18:55:18.020Z 16 Acked 1015 Read 3 Done Sent Done false
138442023-12-10T18:55:18.020Z 17 Acked 1016 Read 4 Done Sent Done false
138452023-12-10T18:55:18.020Z 18 Acked 1017 Flush 0 Done Sent Done false
138462023-12-10T18:55:18.020Z 19 Acked 1018 Write 5 Done Sent Done false
138472023-12-10T18:55:18.020Z 20 Acked 1019 Write 2 Done Sent Done false
138482023-12-10T18:55:18.021Z 21 Acked 1020 Flush 0 Done Sent Done false
138492023-12-10T18:55:18.021Z 22 Acked 1021 Write 5 Done Sent Done false
138502023-12-10T18:55:18.021Z 23 Acked 1022 Write 6 Done Sent Done false
138512023-12-10T18:55:18.021Z 24 Acked 1023 Read 2 Done Sent Done false
138522023-12-10T18:55:18.021Z 25 Acked 1024 Read 3 Done Sent Done false
138532023-12-10T18:55:18.021Z 26 Acked 1025 Write 2 Done Sent Done false
138542023-12-10T18:55:18.021Z 27 Acked 1026 Write 10 Done Sent Done false
138552023-12-10T18:55:18.021Z 28 Acked 1027 Write 3 Done Sent Done false
138562023-12-10T18:55:18.021Z 29 Acked 1028 Read 9 Done Sent Done false
138572023-12-10T18:55:18.021Z 30 Acked 1029 Read 2 Done Sent Done false
138582023-12-10T18:55:18.022Z STATES DS:0 DS:1 DS:2 TOTAL
138592023-12-10T18:55:18.022Z New 0 0 0 0
138602023-12-10T18:55:18.022Z Sent 0 26 0 26
138612023-12-10T18:55:18.022Z Done 30 4 30 64
138622023-12-10T18:55:18.022Z Skipped 0 0 0 0
138632023-12-10T18:55:18.022Z Error 0 0 0 0
138642023-12-10T18:55:18.022ZLast Flush: 1020 0 1020
138652023-12-10T18:55:18.022ZDownstairs last five completed:
138662023-12-10T18:55:18.023ZUpstairs last five completed: 30 29 28 27 26 25
138672023-12-10T18:55:18.023ZWrote out file "/tmp/test_repair_verify.data"
138682023-12-10T18:55:18.023Z
138692023-12-10T18:55:18.023ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
138702023-12-10T18:55:18.024ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
138712023-12-10T18:55:18.024Z 0 000-019 100 100 100 179 179 179 F F F
138722023-12-10T18:55:18.024Z 1 020-039 108 108 108 191 191 191 F F F
138732023-12-10T18:55:18.024Z 2 040-059 108 108 108 190 190 190 F F F
138742023-12-10T18:55:18.024Z 3 060-079 112 110 112 198 195 198 T F T <---
138752023-12-10T18:55:18.024Z 4 080-099 108 108 108 189 189 189 F F F
138762023-12-10T18:55:18.024Z 5 100-119 112 110 112 198 195 198 F F F <---
138772023-12-10T18:55:18.024Z 6 120-139 108 108 108 190 190 190 F F F
138782023-12-10T18:55:18.024Z 7 140-159 112 110 112 199 193 199 F F F <---
138792023-12-10T18:55:18.024Z 8 160-179 111 111 111 196 196 196 F F F
138802023-12-10T18:55:18.025Z 9 180-199 107 107 107 188 188 188 F F F
138812023-12-10T18:55:18.025Z 10 200-219 112 110 112 197 194 197 F T F <---
138822023-12-10T18:55:18.025Z 11 220-239 112 110 112 199 195 199 F F F <---
138832023-12-10T18:55:18.025Z 12 240-259 110 110 110 195 195 195 F F F
138842023-12-10T18:55:18.025Z 13 260-279 110 110 110 195 195 195 T F T <---
138852023-12-10T18:55:18.025Z 14 280-299 111 111 111 196 196 196 F F F
138862023-12-10T18:55:18.025Z 15 300-319 112 110 112 199 193 199 F F F <---
138872023-12-10T18:55:18.025Z 16 320-339 112 105 112 199 184 199 F F F <---
138882023-12-10T18:55:18.025Z 17 340-359 112 103 112 200 182 200 F F F <---
138892023-12-10T18:55:18.025Z 18 360-379 112 107 112 199 188 199 F F F <---
138902023-12-10T18:55:18.026Z 19 380-399 112 107 112 198 188 198 F F F <---
138912023-12-10T18:55:18.026Z 20 400-419 112 110 112 198 195 198 F F F <---
138922023-12-10T18:55:18.026Z 21 420-439 108 108 108 189 189 189 F F F
138932023-12-10T18:55:18.026Z 22 440-459 112 96 112 200 174 200 F F F <---
138942023-12-10T18:55:18.027Z 23 460-479 104 104 104 183 183 183 F F F
138952023-12-10T18:55:18.027Z 24 480-499 111 111 111 196 196 196 F F F
138962023-12-10T18:55:18.027Z 25 500-519 106 106 106 186 186 186 F F F
138972023-12-10T18:55:18.027Z 26 520-539 109 109 109 192 192 192 T F T <---
138982023-12-10T18:55:18.027Z 27 540-559 109 109 109 192 192 192 F F F
138992023-12-10T18:55:18.027Z 28 560-579 110 110 110 195 195 195 F F F
139002023-12-10T18:55:18.027Z 29 580-599 111 111 111 196 196 196 T F T <---
139012023-12-10T18:55:18.027ZMax gen: 112, Max flush: 200
139022023-12-10T18:55:18.027ZError: Difference in extent metadata found!
139032023-12-10T18:55:18.027ZA Difference in extent metadata is expected here
139042023-12-10T18:55:18.028ZOn loop 56
139052023-12-10T18:55:18.028Z
139062023-12-10T18:55:18.028ZVerifying data now
139072023-12-10T18:55:23.074ZNo change after verify
139082023-12-10T18:55:23.074ZLoop: 56 Downstairs dump after verify (and repair):
139092023-12-10T18:55:23.081ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
139102023-12-10T18:55:23.081Z 0 000-019 100 100 100 179 179 179 F F F
139112023-12-10T18:55:23.087Z 1 020-039 108 108 108 191 191 191 F F F
139122023-12-10T18:55:23.087Z 2 040-059 108 108 108 190 190 190 F F F
139132023-12-10T18:55:23.087Z 3 060-079 113 113 113 201 201 201 F F F
139142023-12-10T18:55:23.087Z 4 080-099 108 108 108 189 189 189 F F F
139152023-12-10T18:55:23.087Z 5 100-119 112 112 112 198 198 198 F F F
139162023-12-10T18:55:23.087Z 6 120-139 108 108 108 190 190 190 F F F
139172023-12-10T18:55:23.087Z 7 140-159 112 112 112 199 199 199 F F F
139182023-12-10T18:55:23.087Z 8 160-179 111 111 111 196 196 196 F F F
139192023-12-10T18:55:23.087Z 9 180-199 107 107 107 188 188 188 F F F
139202023-12-10T18:55:23.087Z 10 200-219 112 112 112 197 197 197 F F F
139212023-12-10T18:55:23.087Z 11 220-239 112 112 112 199 199 199 F F F
139222023-12-10T18:55:23.087Z 12 240-259 110 110 110 195 195 195 F F F
139232023-12-10T18:55:23.087Z 13 260-279 113 113 113 201 201 201 F F F
139242023-12-10T18:55:23.087Z 14 280-299 111 111 111 196 196 196 F F F
139252023-12-10T18:55:23.087Z 15 300-319 112 112 112 199 199 199 F F F
139262023-12-10T18:55:23.088Z 16 320-339 112 112 112 199 199 199 F F F
139272023-12-10T18:55:23.088Z 17 340-359 112 112 112 200 200 200 F F F
139282023-12-10T18:55:23.088Z 18 360-379 112 112 112 199 199 199 F F F
139292023-12-10T18:55:23.088Z 19 380-399 112 112 112 198 198 198 F F F
139302023-12-10T18:55:23.088Z 20 400-419 112 112 112 198 198 198 F F F
139312023-12-10T18:55:23.088Z 21 420-439 108 108 108 189 189 189 F F F
139322023-12-10T18:55:23.088Z 22 440-459 112 112 112 200 200 200 F F F
139332023-12-10T18:55:23.088Z 23 460-479 104 104 104 183 183 183 F F F
139342023-12-10T18:55:23.088Z 24 480-499 111 111 111 196 196 196 F F F
139352023-12-10T18:55:23.088Z 25 500-519 106 106 106 186 186 186 F F F
139362023-12-10T18:55:23.088Z 26 520-539 113 113 113 201 201 201 F F F
139372023-12-10T18:55:23.088Z 27 540-559 109 109 109 192 192 192 F F F
139382023-12-10T18:55:23.088Z 28 560-579 110 110 110 195 195 195 F F F
139392023-12-10T18:55:23.088Z 29 580-599 113 113 113 201 201 201 F F F
139402023-12-10T18:55:23.088ZMax gen: 113, Max flush: 201
139412023-12-10T18:55:23.088Z
139422023-12-10T18:55:23.088ZBegin loop 57
139432023-12-10T18:55:23.089ZDownstairs to restart: 1
139442023-12-10T18:55:23.115Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.121858184Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618}
139452023-12-10T18:55:23.115Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.122265848Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618}
139462023-12-10T18:55:23.119Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.122294796Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618}
139472023-12-10T18:55:23.119Z{"msg":"Crucible 5879dad6-a59c-4fe6-8395-06db8b369285 has session id: b8338f98-ae30-4c66-914e-09b484629745","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.122373675Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139482023-12-10T18:55:23.120Z{"msg":"Upstairs opts: Upstairs UUID: 5879dad6-a59c-4fe6-8395-06db8b369285, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
139492023-12-10T18:55:23.120Z,"time":"2023-12-10T18:55:23.122434731Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139502023-12-10T18:55:23.120Z{"msg":"Crucible stats registered with UUID: 5879dad6-a59c-4fe6-8395-06db8b369285","v":0,The guest has requested activation with gen:114
139512023-12-10T18:55:23.120Z"name":"crucible","level":30,"time":"2023-12-10T18:55:23.122758166Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139522023-12-10T18:55:23.120Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.122825253Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745","task":"up_listen"}
139532023-12-10T18:55:23.121Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.122870292Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139542023-12-10T18:55:23.121Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.122925693Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139552023-12-10T18:55:23.121Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.1229655Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"looper":"2","session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139562023-12-10T18:55:23.121Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.123103082Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"looper":"0","session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139572023-12-10T18:55:23.121Z{"msg":"[2] 5879dad6-a59c-4fe6-8395-06db8b369285 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.123180917Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"looper":"2","session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139582023-12-10T18:55:23.121Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.123304654Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"looper":"1","session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139592023-12-10T18:55:23.121Z{"msg":"[0] 5879dad6-a59c-4fe6-8395-06db8b369285 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.123354479Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"looper":"0","session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139602023-12-10T18:55:23.121Z{"msg":"[1] 5879dad6-a59c-4fe6-8395-06db8b369285 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.123430158Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"looper":"1","session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139612023-12-10T18:55:23.121Z{"msg":"5879dad6-a59c-4fe6-8395-06db8b369285 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.123813276Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139622023-12-10T18:55:23.122Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.123968022Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139632023-12-10T18:55:23.122Z{"msg":"Set desired generation to :114","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.124040739Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139642023-12-10T18:55:23.122Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.124209064Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139652023-12-10T18:55:23.122Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.124291141Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139662023-12-10T18:55:23.122Z{"msg":"[2] 5879dad6-a59c-4fe6-8395-06db8b369285 (b8338f98-ae30-4c66-914e-09b484629745) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.124663977Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139672023-12-10T18:55:23.122Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.124747729Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139682023-12-10T18:55:23.122Z{"msg":"[2] client is_active_req TRUE, promote! session b8338f98-ae30-4c66-914e-09b484629745","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.124820207Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139692023-12-10T18:55:23.122Z{"msg":"[0] 5879dad6-a59c-4fe6-8395-06db8b369285 (b8338f98-ae30-4c66-914e-09b484629745) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.124895166Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139702023-12-10T18:55:23.122Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.124966164Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139712023-12-10T18:55:23.122Z{"msg":"[0] client is_active_req TRUE, promote! session b8338f98-ae30-4c66-914e-09b484629745","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.125026475Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139722023-12-10T18:55:23.122Z{"msg":"[1] 5879dad6-a59c-4fe6-8395-06db8b369285 (b8338f98-ae30-4c66-914e-09b484629745) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.125230083Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139732023-12-10T18:55:23.122Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.125365693Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139742023-12-10T18:55:23.122Z{"msg":"[1] client is_active_req TRUE, promote! session b8338f98-ae30-4c66-914e-09b484629745","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.1253919Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139752023-12-10T18:55:23.123Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.12542022Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139762023-12-10T18:55:23.123Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.125494496Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139772023-12-10T18:55:23.123Z{"msg":"5879dad6-a59c-4fe6-8395-06db8b369285 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.125532656Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139782023-12-10T18:55:23.128Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.125674295Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139792023-12-10T18:55:23.128Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.125789472Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139802023-12-10T18:55:23.128Z{"msg":"5879dad6-a59c-4fe6-8395-06db8b369285 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.125829623Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139812023-12-10T18:55:23.130Z{"msg":"[2] 5879dad6-a59c-4fe6-8395-06db8b369285 (b8338f98-ae30-4c66-914e-09b484629745) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.126225949Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139822023-12-10T18:55:23.130Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.126299923Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139832023-12-10T18:55:23.130Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:23.126330081Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139842023-12-10T18:55:23.130Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.126418745Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139852023-12-10T18:55:23.130Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.126480141Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139862023-12-10T18:55:23.130Z{"msg":"5879dad6-a59c-4fe6-8395-06db8b369285 WaitActive WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.126503952Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139872023-12-10T18:55:23.130Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.126615696Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139882023-12-10T18:55:23.130Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.126661886Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139892023-12-10T18:55:23.130Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.126685831Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139902023-12-10T18:55:23.130Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.126711262Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139912023-12-10T18:55:23.131Z{"msg":"5879dad6-a59c-4fe6-8395-06db8b369285 WaitActive WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.12675335Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139922023-12-10T18:55:23.132Z{"msg":"[0] 5879dad6-a59c-4fe6-8395-06db8b369285 (b8338f98-ae30-4c66-914e-09b484629745) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.126799449Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139932023-12-10T18:55:23.132Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.126845088Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139942023-12-10T18:55:23.132Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:23.126867702Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139952023-12-10T18:55:23.132Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.12688759Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139962023-12-10T18:55:23.132Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.126928736Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139972023-12-10T18:55:23.132Z{"msg":"5879dad6-a59c-4fe6-8395-06db8b369285 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.126952042Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139982023-12-10T18:55:23.132Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.126972607Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
139992023-12-10T18:55:23.132Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.127011871Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140002023-12-10T18:55:23.132Z{"msg":"[1] 5879dad6-a59c-4fe6-8395-06db8b369285 (b8338f98-ae30-4c66-914e-09b484629745) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.127393274Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140012023-12-10T18:55:23.132Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.127463183Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140022023-12-10T18:55:23.132Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:23.127490027Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140032023-12-10T18:55:23.132Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.127514729Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140042023-12-10T18:55:23.132Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.127588284Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140052023-12-10T18:55:23.132Z{"msg":"5879dad6-a59c-4fe6-8395-06db8b369285 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.127629634Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140062023-12-10T18:55:23.132Z{"msg":"[0]R flush_numbers[0..12]: [179, 191, 190, 201, 189, 198, 190, 199, 196, 188, 197, 199]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.127687125Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140072023-12-10T18:55:23.132Z{"msg":"[0]R generation[0..12]: [100, 108, 108, 113, 108, 112, 108, 112, 111, 107, 112, 112]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.127728573Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140082023-12-10T18:55:23.133Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0The guest has finished waiting for activation with:114
140092023-12-10T18:55:23.133Z,"name":"crucible","level"Wait for a query_work_queue command to finish before sending IO
140102023-12-10T18:55:23.133Z:30,"time":"2023-12-10T18:55:23.127790765Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140112023-12-10T18:55:23.133Z{"msg":"[1]R flush_numbers[0..12]: [179, 191, 190, 201, 189, 198, 190, 199, 196, 188, 197, 199]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.127832381Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140122023-12-10T18:55:23.133Z{"msg":"[1]R generation[0..12]: [100, 108, 108, 113, 108, 112, 108, 112, 111, 107, 112, 112]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.127903333Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140132023-12-10T18:55:23.133Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.127995465Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140142023-12-10T18:55:23.133Z{"msg":"[2]R flush_numbers[0..12]: [179, 191, 190, 201, 189, 198, 190, 199, 196, 188, 197, 199]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.128036886Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140152023-12-10T18:55:23.134Z{"msg":"[2]R generation[0..12]: [100, 108, 108, 113, 108, 112, 108, 112, 111, 107, 112, 112]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.128101236Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140162023-12-10T18:55:23.134Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.128128426Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140172023-12-10T18:55:23.134Z{"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.128148421Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140182023-12-10T18:55:23.134Z{"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.128211544Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140192023-12-10T18:55:23.134Z{"msg":"Next flush: 202","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.128251715Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140202023-12-10T18:55:23.134Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.128321372Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140212023-12-10T18:55:23.134Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.128360585Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140222023-12-10T18:55:23.136Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.128420374Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140232023-12-10T18:55:23.136Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.128444857Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140242023-12-10T18:55:23.136Z{"msg":"5879dad6-a59c-4fe6-8395-06db8b369285 is now active with session: b8338f98-ae30-4c66-914e-09b484629745","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.12847659Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140252023-12-10T18:55:23.137Z{"msg":"5879dad6-a59c-4fe6-8395-06db8b369285 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.12855408Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140262023-12-10T18:55:23.137Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.128585278Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140272023-12-10T18:55:23.137Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.128606889Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140282023-12-10T18:55:23.137Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
140292023-12-10T18:55:23.137Z,"time":"2023-12-10T18:55:23.128666868Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140302023-12-10T18:55:23.137Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.12871654Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140312023-12-10T18:55:23.137Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.128738454Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140322023-12-10T18:55:23.137Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.128778323Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140332023-12-10T18:55:23.138Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.128799769Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140342023-12-10T18:55:23.138Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.128839252Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140352023-12-10T18:55:23.138Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.128860746Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140362023-12-10T18:55:23.138Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.12889987Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140372023-12-10T18:55:23.138Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:23.128921222Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1618,"session_id":"b8338f98-ae30-4c66-914e-09b484629745"}
140382023-12-10T18:55:23.138ZLoading write count information from file "/tmp/test_repair_verify.data"
140392023-12-10T18:55:23.138ZRun Repair workload
140402023-12-10T18:55:23.138Z01/30 Write block 326 len 36864 data: 9 10 9 8 9 9 11 11 9
140412023-12-10T18:55:23.138Z02/30 Write block 559 len 8192 data: 7 7
140422023-12-10T18:55:23.139Z03/30 Read block 451 len 20480
140432023-12-10T18:55:23.144Z04/30 Read block 504 len 12288
140442023-12-10T18:55:23.145Z05/30 Read block 386 len 40960
140452023-12-10T18:55:23.145Z06/30 Write block 520 len 20480 data: 6 7 6 7 8
140462023-12-10T18:55:23.145Z07/30 Write block 36 len 4096 data: 11
140472023-12-10T18:55:23.145Z08/30 Read block 158 len 20480
140482023-12-10T18:55:23.145Z09/30 Read block 129 len 8192
140492023-12-10T18:55:23.145Z10/30 Write block 249 len 28672 data: 13 12 12 12 13 10 8
140502023-12-10T18:55:23.145Z11/30 Read block 476 len 40960
140512023-12-10T18:55:23.145Z12/30 Write block 168 len 8192 data: 12 10
140522023-12-10T18:55:23.554Z13/30 Read block 572 len 28672
140532023-12-10T18:55:23.554Z14/30 Read block 408 len 36864
140542023-12-10T18:55:23.555Z15/30 Read block 565 len 32768
140552023-12-10T18:55:23.555Z16/30 Read block 552 len 28672
140562023-12-10T18:55:23.555Z17/30 Read block 107 len 20480
140572023-12-10T18:55:23.555Z18/30 Read block 91 len 32768
140582023-12-10T18:55:23.555Z19/30 Write block 188 len 20480 data: 6 6 9 10 9
140592023-12-10T18:55:23.555Z20/30 Write block 379 len 16384 data: 15 16 12 13
140602023-12-10T18:55:23.555Z21/30 Write block 452 len 32768 data: 9 9 7 8 8 8 9 7
140612023-12-10T18:55:23.555Z22/30 Write block 171 len 4096 data: 10
140622023-12-10T18:55:23.555Z23/30 Read block 483 len 16384
140632023-12-10T18:55:23.555Z24/30 Read block 244 len 28672
140642023-12-10T18:55:23.555Z25/30 Read block 199 len 20480
140652023-12-10T18:55:23.555Z26/30 Read block 122 len 8192
140662023-12-10T18:55:23.555Z27/30 Write block 474 len 16384 data: 8 7 7 5
140672023-12-10T18:55:23.555Z28/30 Read block 66 len 8192
140682023-12-10T18:55:23.555Z29/30 Write block 386 len 8192 data: 12 11
140692023-12-10T18:55:23.555Z30/30 Write block 282 len 36864 data: 8 9 10 13 13 13 12 12 9
140702023-12-10T18:55:23.555Z----------------------------------------------------------------
140712023-12-10T18:55:23.555Z Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30
140722023-12-10T18:55:23.556ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
140732023-12-10T18:55:23.556Z 1 Acked 1000 Write 9 Done Done Done false
140742023-12-10T18:55:23.556Z 2 Acked 1001 Write 2 Done Done Done false
140752023-12-10T18:55:23.556Z 3 Acked 1002 Read 5 Done Done Done false
140762023-12-10T18:55:23.556Z 4 Acked 1003 Read 3 Done Done Done false
140772023-12-10T18:55:23.556Z 5 Acked 1004 Read 10 Done Done Done false
140782023-12-10T18:55:23.556Z 6 Acked 1005 Write 5 Done Done Done false
140792023-12-10T18:55:23.556Z 7 Acked 1006 Write 1 Done Done Done false
140802023-12-10T18:55:23.556Z 8 Acked 1007 Read 5 Done Done Done false
140812023-12-10T18:55:23.556Z 9 Acked 1008 Read 2 Done Done Done false
140822023-12-10T18:55:23.556Z 10 Acked 1009 Write 7 Done Done Done false
140832023-12-10T18:55:23.556Z 11 Acked 1010 Read 10 Done Done Done false
140842023-12-10T18:55:23.556Z 12 Acked 1011 Write 2 Done Done Done false
140852023-12-10T18:55:23.556Z 13 Acked 1012 Read 7 Done Sent Done false
140862023-12-10T18:55:23.556Z 14 Acked 1013 Read 9 Done Sent Done false
140872023-12-10T18:55:23.556Z 15 Acked 1014 Read 8 Done Sent Done false
140882023-12-10T18:55:23.557Z 16 Acked 1015 Read 7 Done Sent Done false
140892023-12-10T18:55:23.557Z 17 Acked 1016 Read 5 Done Sent Done false
140902023-12-10T18:55:23.557Z 18 Acked 1017 Read 8 Done Sent Done false
140912023-12-10T18:55:23.557Z 19 Acked 1018 Write 5 Done Sent Done false
140922023-12-10T18:55:23.557Z 20 Acked 1019 Write 4 Done Sent Done false
140932023-12-10T18:55:23.557Z 21 Acked 1020 Write 8 Done Sent Done false
140942023-12-10T18:55:23.557Z 22 Acked 1021 Write 1 Done Sent Done false
140952023-12-10T18:55:23.557Z 23 Acked 1022 Read 4 Done Sent Done false
140962023-12-10T18:55:23.557Z 24 Acked 1023 Read 7 Done Sent Done false
140972023-12-10T18:55:23.557Z 25 Acked 1024 Read 5 Done Sent Done false
140982023-12-10T18:55:23.557Z 26 Acked 1025 Read 2 Done Sent Done false
140992023-12-10T18:55:23.557Z 27 Acked 1026 Write 4 Done Sent Done false
141002023-12-10T18:55:23.557Z 28 Acked 1027 Read 2 Done Sent Done false
141012023-12-10T18:55:23.557Z 29 Acked 1028 Write 2 Done Sent Done false
141022023-12-10T18:55:23.557Z 30 Acked 1029 Write 9 Sent Sent Sent false
141032023-12-10T18:55:23.557Z STATES DS:0 DS:1 DS:2 TOTAL
141042023-12-10T18:55:23.557Z New 0 0 0 0
141052023-12-10T18:55:23.558Z Sent 1 18 1 20
141062023-12-10T18:55:23.558Z Done 29 12 29 70
141072023-12-10T18:55:23.559Z Skipped 0 0 0 0
141082023-12-10T18:55:23.559Z Error 0 0 0 0
141092023-12-10T18:55:23.559ZLast Flush: 0 0 0
141102023-12-10T18:55:23.559ZDownstairs last five completed:
141112023-12-10T18:55:23.559ZUpstairs last five completed: 30 29 28 27 26 25
141122023-12-10T18:55:23.559ZWrote out file "/tmp/test_repair_verify.data"
141132023-12-10T18:55:23.559Z
141142023-12-10T18:55:23.559ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
141152023-12-10T18:55:23.559ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
141162023-12-10T18:55:23.559Z 0 000-019 100 100 100 179 179 179 F F F
141172023-12-10T18:55:23.559Z 1 020-039 108 108 108 191 191 191 T T T
141182023-12-10T18:55:23.559Z 2 040-059 108 108 108 190 190 190 F F F
141192023-12-10T18:55:23.560Z 3 060-079 113 113 113 201 201 201 F F F
141202023-12-10T18:55:23.560Z 4 080-099 108 108 108 189 189 189 F F F
141212023-12-10T18:55:23.560Z 5 100-119 112 112 112 198 198 198 F F F
141222023-12-10T18:55:23.560Z 6 120-139 108 108 108 190 190 190 F F F
141232023-12-10T18:55:23.560Z 7 140-159 112 112 112 199 199 199 F F F
141242023-12-10T18:55:23.560Z 8 160-179 111 111 111 196 196 196 T T T
141252023-12-10T18:55:23.560Z 9 180-199 107 107 107 188 188 188 T F T <---
141262023-12-10T18:55:23.560Z 10 200-219 112 112 112 197 197 197 F F F
141272023-12-10T18:55:23.560Z 11 220-239 112 112 112 199 199 199 F F F
141282023-12-10T18:55:23.560Z 12 240-259 110 110 110 195 195 195 T T T
141292023-12-10T18:55:23.560Z 13 260-279 113 113 113 201 201 201 F F F
141302023-12-10T18:55:23.560Z 14 280-299 111 111 111 196 196 196 T F T <---
141312023-12-10T18:55:23.560Z 15 300-319 112 112 112 199 199 199 F F F
141322023-12-10T18:55:23.560Z 16 320-339 112 112 112 199 199 199 T T T
141332023-12-10T18:55:23.560Z 17 340-359 112 112 112 200 200 200 F F F
141342023-12-10T18:55:23.560Z 18 360-379 112 112 112 199 199 199 T F T <---
141352023-12-10T18:55:23.560Z 19 380-399 112 112 112 198 198 198 T F T <---
141362023-12-10T18:55:23.560Z 20 400-419 112 112 112 198 198 198 F F F
141372023-12-10T18:55:23.561Z 21 420-439 108 108 108 189 189 189 F F F
141382023-12-10T18:55:23.561Z 22 440-459 112 112 112 200 200 200 T F T <---
141392023-12-10T18:55:23.561Z 23 460-479 104 104 104 183 183 183 T F T <---
141402023-12-10T18:55:23.561Z 24 480-499 111 111 111 196 196 196 F F F
141412023-12-10T18:55:23.561Z 25 500-519 106 106 106 186 186 186 F F F
141422023-12-10T18:55:23.561Z 26 520-539 113 113 113 201 201 201 T T T
141432023-12-10T18:55:23.561Z 27 540-559 109 109 109 192 192 192 T T T
141442023-12-10T18:55:23.561Z 28 560-579 110 110 110 195 195 195 T T T
141452023-12-10T18:55:23.561Z 29 580-599 113 113 113 201 201 201 F F F
141462023-12-10T18:55:23.561ZMax gen: 113, Max flush: 201
141472023-12-10T18:55:23.561ZError: Difference in extent metadata found!
141482023-12-10T18:55:23.561ZA Difference in extent metadata is expected here
141492023-12-10T18:55:23.562ZOn loop 57
141502023-12-10T18:55:23.563Z
141512023-12-10T18:55:23.563ZVerifying data now
141522023-12-10T18:55:28.516ZNo change after verify
141532023-12-10T18:55:28.516ZLoop: 57 Downstairs dump after verify (and repair):
141542023-12-10T18:55:28.567ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
141552023-12-10T18:55:28.567Z 0 000-019 100 100 100 179 179 179 F F F
141562023-12-10T18:55:28.574Z 1 020-039 114 114 114 202 202 202 F F F
141572023-12-10T18:55:28.574Z 2 040-059 108 108 108 190 190 190 F F F
141582023-12-10T18:55:28.574Z 3 060-079 113 113 113 201 201 201 F F F
141592023-12-10T18:55:28.574Z 4 080-099 108 108 108 189 189 189 F F F
141602023-12-10T18:55:28.575Z 5 100-119 112 112 112 198 198 198 F F F
141612023-12-10T18:55:28.575Z 6 120-139 108 108 108 190 190 190 F F F
141622023-12-10T18:55:28.575Z 7 140-159 112 112 112 199 199 199 F F F
141632023-12-10T18:55:28.575Z 8 160-179 114 114 114 202 202 202 F F F
141642023-12-10T18:55:28.575Z 9 180-199 114 114 114 202 202 202 F F F
141652023-12-10T18:55:28.575Z 10 200-219 112 112 112 197 197 197 F F F
141662023-12-10T18:55:28.575Z 11 220-239 112 112 112 199 199 199 F F F
141672023-12-10T18:55:28.575Z 12 240-259 114 114 114 202 202 202 F F F
141682023-12-10T18:55:28.576Z 13 260-279 113 113 113 201 201 201 F F F
141692023-12-10T18:55:28.576Z 14 280-299 114 114 114 202 202 202 F F F
141702023-12-10T18:55:28.576Z 15 300-319 112 112 112 199 199 199 F F F
141712023-12-10T18:55:28.576Z 16 320-339 114 114 114 202 202 202 F F F
141722023-12-10T18:55:28.576Z 17 340-359 112 112 112 200 200 200 F F F
141732023-12-10T18:55:28.576Z 18 360-379 114 114 114 202 202 202 F F F
141742023-12-10T18:55:28.576Z 19 380-399 114 114 114 202 202 202 F F F
141752023-12-10T18:55:28.576Z 20 400-419 112 112 112 198 198 198 F F F
141762023-12-10T18:55:28.576Z 21 420-439 108 108 108 189 189 189 F F F
141772023-12-10T18:55:28.576Z 22 440-459 114 114 114 202 202 202 F F F
141782023-12-10T18:55:28.576Z 23 460-479 114 114 114 202 202 202 F F F
141792023-12-10T18:55:28.576Z 24 480-499 111 111 111 196 196 196 F F F
141802023-12-10T18:55:28.576Z 25 500-519 106 106 106 186 186 186 F F F
141812023-12-10T18:55:28.577Z 26 520-539 114 114 114 202 202 202 F F F
141822023-12-10T18:55:28.577Z 27 540-559 114 114 114 202 202 202 F F F
141832023-12-10T18:55:28.577Z 28 560-579 114 114 114 202 202 202 F F F
141842023-12-10T18:55:28.577Z 29 580-599 113 113 113 201 201 201 F F F
141852023-12-10T18:55:28.577ZMax gen: 114, Max flush: 202
141862023-12-10T18:55:28.577Z
141872023-12-10T18:55:28.577ZBegin loop 58
141882023-12-10T18:55:28.577ZDownstairs to restart: 1
141892023-12-10T18:55:28.577Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.574081982Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626}
141902023-12-10T18:55:28.577Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.574451218Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626}
141912023-12-10T18:55:28.577Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.57451091Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626}
141922023-12-10T18:55:28.577Z{"msg":"Crucible cff527a6-f4a2-4a65-a995-58fc523524ce has session id: 4a0a0983-1ee4-482f-97d2-e77c4f16a107","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.574551273Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
141932023-12-10T18:55:28.578Z{"msg":"Upstairs opts: Upstairs UUID: cff527a6-f4a2-4a65-a995-58fc523524ce, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.574637306Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
141942023-12-10T18:55:28.578Z{"msg":"Crucible stats registered with UUID: cff527a6-f4a2-4a65-a995-58fc523524ce","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.574680485Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
141952023-12-10T18:55:28.578ZCrucible runtime is spawned
141962023-12-10T18:55:28.578ZThe guest has requested activation with gen:116
141972023-12-10T18:55:28.578Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.57496496Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107","task":"up_listen"}
141982023-12-10T18:55:28.579Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.575051103Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
141992023-12-10T18:55:28.579Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.575092226Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142002023-12-10T18:55:28.579Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.57519935Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"looper":"1","session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142012023-12-10T18:55:28.579Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.575299202Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"looper":"0","session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142022023-12-10T18:55:28.579Z{"msg":"[1] cff527a6-f4a2-4a65-a995-58fc523524ce looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.575430861Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"looper":"1","session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142032023-12-10T18:55:28.580Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.575535314Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"looper":"2","session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142042023-12-10T18:55:28.580Z{"msg":"[0] cff527a6-f4a2-4a65-a995-58fc523524ce looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.57567084Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"looper":"0","session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142052023-12-10T18:55:28.580Z{"msg":"[2] cff527a6-f4a2-4a65-a995-58fc523524ce looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.575715764Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"looper":"2","session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142062023-12-10T18:55:28.580Z{"msg":"cff527a6-f4a2-4a65-a995-58fc523524ce active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.576018669Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142072023-12-10T18:55:28.580Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.576077967Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142082023-12-10T18:55:28.580Z{"msg":"Set desired generation to :116","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.576101634Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142092023-12-10T18:55:28.580Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.57635534Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142102023-12-10T18:55:28.580Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.576411303Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142112023-12-10T18:55:28.580Z{"msg":"[0] cff527a6-f4a2-4a65-a995-58fc523524ce (4a0a0983-1ee4-482f-97d2-e77c4f16a107) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.576804013Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142122023-12-10T18:55:28.580Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.576923464Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142132023-12-10T18:55:28.581Z{"msg":"[0] client is_active_req TRUE, promote! session 4a0a0983-1ee4-482f-97d2-e77c4f16a107","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.57697311Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142142023-12-10T18:55:28.581Z{"msg":"[2] cff527a6-f4a2-4a65-a995-58fc523524ce (4a0a0983-1ee4-482f-97d2-e77c4f16a107) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.577058508Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142152023-12-10T18:55:28.581Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.577103477Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142162023-12-10T18:55:28.581Z{"msg":"[2] client is_active_req TRUE, promote! session 4a0a0983-1ee4-482f-97d2-e77c4f16a107","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.577200118Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142172023-12-10T18:55:28.581Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.577707032Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142182023-12-10T18:55:28.581Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.577775389Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142192023-12-10T18:55:28.581Z{"msg":"cff527a6-f4a2-4a65-a995-58fc523524ce WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.57780792Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142202023-12-10T18:55:28.581Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.577991211Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142212023-12-10T18:55:28.581Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.578068249Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142222023-12-10T18:55:28.581Z{"msg":"cff527a6-f4a2-4a65-a995-58fc523524ce WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.57809585Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142232023-12-10T18:55:28.581Z{"msg":"[1] cff527a6-f4a2-4a65-a995-58fc523524ce (4a0a0983-1ee4-482f-97d2-e77c4f16a107) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.578128553Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142242023-12-10T18:55:28.589Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.578151089Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142252023-12-10T18:55:28.589Z{"msg":"[1] client is_active_req TRUE, promote! session 4a0a0983-1ee4-482f-97d2-e77c4f16a107","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.578206892Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142262023-12-10T18:55:28.589Z{"msg":"[0] cff527a6-f4a2-4a65-a995-58fc523524ce (4a0a0983-1ee4-482f-97d2-e77c4f16a107) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.578520574Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142272023-12-10T18:55:28.589Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.578604349Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142282023-12-10T18:55:28.590Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:28.578640106Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142292023-12-10T18:55:28.590Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.578701056Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142302023-12-10T18:55:28.590Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.578730841Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142312023-12-10T18:55:28.590Z{"msg":"cff527a6-f4a2-4a65-a995-58fc523524ce WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.578786511Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142322023-12-10T18:55:28.590Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.578816214Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142332023-12-10T18:55:28.590Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.578901191Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142342023-12-10T18:55:28.590Z{"msg":"[2] cff527a6-f4a2-4a65-a995-58fc523524ce (4a0a0983-1ee4-482f-97d2-e77c4f16a107) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.578934532Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142352023-12-10T18:55:28.590Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.578955616Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142362023-12-10T18:55:28.590Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:28.578990377Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142372023-12-10T18:55:28.591Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.579016979Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142382023-12-10T18:55:28.591Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.579075126Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142392023-12-10T18:55:28.591Z{"msg":"cff527a6-f4a2-4a65-a995-58fc523524ce WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.579148524Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142402023-12-10T18:55:28.591Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.579184285Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142412023-12-10T18:55:28.591Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.579220908Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142422023-12-10T18:55:28.592Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.579287352Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142432023-12-10T18:55:28.592Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.579367856Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142442023-12-10T18:55:28.592Z{"msg":"cff527a6-f4a2-4a65-a995-58fc523524ce WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.579445313Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142452023-12-10T18:55:28.592Z{"msg":"[1] cff527a6-f4a2-4a65-a995-58fc523524ce (4a0a0983-1ee4-482f-97d2-e77c4f16a107) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.580001544Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142462023-12-10T18:55:28.592Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.580109442Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142472023-12-10T18:55:28.592Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:28.58015376Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142482023-12-10T18:55:28.592Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.580180082Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142492023-12-10T18:55:28.592Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.580225366Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142502023-12-10T18:55:28.592Z{"msg":"cff527a6-f4a2-4a65-a995-58fc523524ce WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.58024977Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142512023-12-10T18:55:28.592Z{"msg":"[0]R flush_numbers[0..12]: [179, 202, 190, 201, 189, 198, 190, 199, 202, 202, 197, 199]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.580271727Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142522023-12-10T18:55:28.593Z{"msg":"[0]R generation[0..12]: [100, 114, 108, 113, 108, 112, 108, 112, 114, 114, 112, 112]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.580314058Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142532023-12-10T18:55:28.593Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.580338636Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142542023-12-10T18:55:28.593Z{"msg":"[1]R flush_numbers[0..12]: [179, 202, 190, 201, 189, 198, 190, 199, 202, 202, 197, 199]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.580360985Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142552023-12-10T18:55:28.593Z{"msg":"[1]R generation[0..12]: [100, 114, 108, 113, 108, 112, 108, 112, 114, 114, 112, 112]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.580402284Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142562023-12-10T18:55:28.593Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.580426673Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142572023-12-10T18:55:28.593Z{"msg":"[2]R flush_numbers[0..12]: [179, 202, 190, 201, 189, 198, 190, 199, 202, 202, 197, 199]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:116
142582023-12-10T18:55:28.593Z,"time":"2023-12-10T18:55:28.580448241Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"Wait for a query_work_queue command to finish before sending IO
142592023-12-10T18:55:28.593Zsession_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142602023-12-10T18:55:28.593Z{"msg":"[2]R generation[0..12]: [100, 114, 108, 113, 108, 112, 108, 112, 114, 114, 112, 112]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.580475178Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142612023-12-10T18:55:28.593Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.580526058Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142622023-12-10T18:55:28.593Z{"msg":"Max found gen is 115","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.580547934Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142632023-12-10T18:55:28.594Z{"msg":"Generation requested: 116 >= found:115","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.580568146Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142642023-12-10T18:55:28.594Z{"msg":"Next flush: 203","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.580587764Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142652023-12-10T18:55:28.594Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.580607347Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142662023-12-10T18:55:28.594Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.580638225Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142672023-12-10T18:55:28.594Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.580715555Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142682023-12-10T18:55:28.594Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.580742628Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142692023-12-10T18:55:28.594Z{"msg":"cff527a6-f4a2-4a65-a995-58fc523524ce is now active with session: 4a0a0983-1ee4-482f-97d2-e77c4f16a107","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.580765948Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142702023-12-10T18:55:28.595Z{"msg":"cff527a6-f4a2-4a65-a995-58fc523524ce Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.580836527Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142712023-12-10T18:55:28.595Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.580865163Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142722023-12-10T18:55:28.595Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.580885396Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142732023-12-10T18:55:28.595Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.580928626Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142742023-12-10T18:55:28.595Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.580951079Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142752023-12-10T18:55:28.596Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.580975728Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142762023-12-10T18:55:28.596Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.58121206Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142772023-12-10T18:55:28.596Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.581274117Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142782023-12-10T18:55:28.596Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.581392804Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142792023-12-10T18:55:28.596Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.581449699Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142802023-12-10T18:55:28.596ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
142812023-12-10T18:55:28.596Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.581611099Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142822023-12-10T18:55:28.596Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:28.581647837Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1626,"session_id":"4a0a0983-1ee4-482f-97d2-e77c4f16a107"}
142832023-12-10T18:55:28.596ZLoading write count information from file "/tmp/test_repair_verify.data"
142842023-12-10T18:55:28.596ZRun Repair workload
142852023-12-10T18:55:28.596Z01/30 Write block 514 len 16384 data: 8 7 7 7
142862023-12-10T18:55:28.597Z02/30 Read block 345 len 12288
142872023-12-10T18:55:28.597Z03/30 Read block 33 len 28672
142882023-12-10T18:55:28.597Z04/30 Read block 410 len 36864
142892023-12-10T18:55:28.597Z05/30 Read block 532 len 24576
142902023-12-10T18:55:28.601Z06/30 Read block 22 len 16384
142912023-12-10T18:55:28.601Z07/30 Write block 86 len 16384 data: 14 11 12 10
142922023-12-10T18:55:28.601Z08/30 Write block 23 len 8192 data: 6 7
142932023-12-10T18:55:28.601Z09/30 Flush
142942023-12-10T18:55:28.602Z10/30 Write block 448 len 8192 data: 9 8
142952023-12-10T18:55:28.602Z11/30 Read block 120 len 20480
142962023-12-10T18:55:28.602Z12/30 Read block 97 len 32768
142972023-12-10T18:55:28.602Z13/30 Read block 515 len 32768
142982023-12-10T18:55:28.607Z14/30 Read block 452 len 36864
142992023-12-10T18:55:28.607Z15/30 Read block 251 len 4096
143002023-12-10T18:55:28.607Z16/30 Read block 100 len 28672
143012023-12-10T18:55:28.611Z17/30 Write block 162 len 12288 data: 14 14 13
143022023-12-10T18:55:28.611Z18/30 Read block 460 len 8192
143032023-12-10T18:55:28.611Z19/30 Read block 303 len 12288
143042023-12-10T18:55:28.614Z20/30 Write block 540 len 8192 data: 12 13
143052023-12-10T18:55:28.615Z21/30 Read block 502 len 40960
143062023-12-10T18:55:28.620Z22/30 Flush
143072023-12-10T18:55:28.620Z23/30 Write block 60 len 16384 data: 4 3 5 6
143082023-12-10T18:55:28.620Z24/30 Read block 541 len 4096
143092023-12-10T18:55:28.630Z25/30 Read block 116 len 36864
143102023-12-10T18:55:28.630Z26/30 Write block 385 len 16384 data: 13 13 12 11
143112023-12-10T18:55:28.630Z27/30 Write block 352 len 12288 data: 13 12 10
143122023-12-10T18:55:28.630Z28/30 Write block 277 len 12288 data: 13 13 9
143132023-12-10T18:55:29.045Z29/30 Write block 295 len 40960 data: 5 7 7 8 11 8 10 9 12 13
143142023-12-10T18:55:29.045Z30/30 Write block 8 len 32768 data: 6 6 8 7 9 12 14 13
143152023-12-10T18:55:29.045Z----------------------------------------------------------------
143162023-12-10T18:55:29.045Z Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30
143172023-12-10T18:55:29.045ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
143182023-12-10T18:55:29.046Z 1 Acked 1000 Write 4 Done Done Done false
143192023-12-10T18:55:29.046Z 2 Acked 1001 Read 3 Done Done Done false
143202023-12-10T18:55:29.046Z 3 Acked 1002 Read 7 Done Done Done false
143212023-12-10T18:55:29.046Z 4 Acked 1003 Read 9 Done Sent Done false
143222023-12-10T18:55:29.046Z 5 Acked 1004 Read 6 Done Sent Done false
143232023-12-10T18:55:29.046Z 6 Acked 1005 Read 4 Done Sent Done false
143242023-12-10T18:55:29.046Z 7 Acked 1006 Write 4 Done Sent Done false
143252023-12-10T18:55:29.047Z 8 Acked 1007 Write 2 Done Sent Done false
143262023-12-10T18:55:29.047Z 9 Acked 1008 Flush 0 Done Sent Done false
143272023-12-10T18:55:29.047Z 10 Acked 1009 Write 2 Done Sent Done false
143282023-12-10T18:55:29.048Z 11 Acked 1010 Read 5 Done Sent Done false
143292023-12-10T18:55:29.048Z 12 Acked 1011 Read 8 Done Sent Done false
143302023-12-10T18:55:29.048Z 13 Acked 1012 Read 8 Done Sent Done false
143312023-12-10T18:55:29.048Z 14 Acked 1013 Read 9 Done Sent Done false
143322023-12-10T18:55:29.048Z 15 Acked 1014 Read 1 Done Sent Done false
143332023-12-10T18:55:29.048Z 16 Acked 1015 Read 7 Done Sent Done false
143342023-12-10T18:55:29.048Z 17 Acked 1016 Write 3 Done Sent Done false
143352023-12-10T18:55:29.048Z 18 Acked 1017 Read 2 Done Sent Done false
143362023-12-10T18:55:29.048Z 19 Acked 1018 Read 3 Done Sent Done false
143372023-12-10T18:55:29.048Z 20 Acked 1019 Write 2 Done Sent Done false
143382023-12-10T18:55:29.049Z 21 Acked 1020 Read 10 Done Sent Done false
143392023-12-10T18:55:29.049Z 22 Acked 1021 Flush 0 Done Sent Done false
143402023-12-10T18:55:29.049Z 23 Acked 1022 Write 4 Done Sent Done false
143412023-12-10T18:55:29.049Z 24 Acked 1023 Read 1 Done Sent Done false
143422023-12-10T18:55:29.049Z 25 Acked 1024 Read 9 Done Sent Done false
143432023-12-10T18:55:29.049Z 26 Acked 1025 Write 4 Done Sent Done false
143442023-12-10T18:55:29.049Z 27 Acked 1026 Write 3 Done Sent Done false
143452023-12-10T18:55:29.049Z 28 Acked 1027 Write 3 Done Sent Done false
143462023-12-10T18:55:29.050Z 29 Acked 1028 Write 10 Done Sent Done false
143472023-12-10T18:55:29.050Z 30 Acked 1029 Write 8 Sent Sent Sent false
143482023-12-10T18:55:29.050Z STATES DS:0 DS:1 DS:2 TOTAL
143492023-12-10T18:55:29.050Z New 0 0 0 0
143502023-12-10T18:55:29.050Z Sent 1 27 1 29
143512023-12-10T18:55:29.050Z Done 29 3 29 61
143522023-12-10T18:55:29.050Z Skipped 0 0 0 0
143532023-12-10T18:55:29.050Z Error 0 0 0 0
143542023-12-10T18:55:29.050ZLast Flush: 1021 0 1021
143552023-12-10T18:55:29.050ZDownstairs last five completed:
143562023-12-10T18:55:29.050ZUpstairs last five completed: 30 29 28 27 26 25
143572023-12-10T18:55:29.051ZWrote out file "/tmp/test_repair_verify.data"
143582023-12-10T18:55:29.051Z
143592023-12-10T18:55:29.051ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
143602023-12-10T18:55:29.051ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
143612023-12-10T18:55:29.051Z 0 000-019 100 100 100 179 179 179 T F T <---
143622023-12-10T18:55:29.051Z 1 020-039 116 114 116 203 202 203 F F F <---
143632023-12-10T18:55:29.051Z 2 040-059 108 108 108 190 190 190 F F F
143642023-12-10T18:55:29.051Z 3 060-079 113 113 113 201 201 201 T F T <---
143652023-12-10T18:55:29.051Z 4 080-099 116 108 116 203 189 203 F F F <---
143662023-12-10T18:55:29.051Z 5 100-119 112 112 112 198 198 198 F F F
143672023-12-10T18:55:29.052Z 6 120-139 108 108 108 190 190 190 F F F
143682023-12-10T18:55:29.052Z 7 140-159 112 112 112 199 199 199 F F F
143692023-12-10T18:55:29.052Z 8 160-179 116 114 116 204 202 204 F F F <---
143702023-12-10T18:55:29.052Z 9 180-199 114 114 114 202 202 202 F F F
143712023-12-10T18:55:29.052Z 10 200-219 112 112 112 197 197 197 F F F
143722023-12-10T18:55:29.052Z 11 220-239 112 112 112 199 199 199 F F F
143732023-12-10T18:55:29.052Z 12 240-259 114 114 114 202 202 202 F F F
143742023-12-10T18:55:29.052Z 13 260-279 113 113 113 201 201 201 T F T <---
143752023-12-10T18:55:29.052Z 14 280-299 114 114 114 202 202 202 T F T <---
143762023-12-10T18:55:29.052Z 15 300-319 112 112 112 199 199 199 T F T <---
143772023-12-10T18:55:29.053Z 16 320-339 114 114 114 202 202 202 F F F
143782023-12-10T18:55:29.053Z 17 340-359 112 112 112 200 200 200 T F T <---
143792023-12-10T18:55:29.053Z 18 360-379 114 114 114 202 202 202 F F F
143802023-12-10T18:55:29.053Z 19 380-399 114 114 114 202 202 202 T F T <---
143812023-12-10T18:55:29.053Z 20 400-419 112 112 112 198 198 198 F F F
143822023-12-10T18:55:29.053Z 21 420-439 108 108 108 189 189 189 F F F
143832023-12-10T18:55:29.053Z 22 440-459 116 114 116 204 202 204 F F F <---
143842023-12-10T18:55:29.053Z 23 460-479 114 114 114 202 202 202 F F F
143852023-12-10T18:55:29.053Z 24 480-499 111 111 111 196 196 196 F F F
143862023-12-10T18:55:29.053Z 25 500-519 116 106 116 203 186 203 F T F <---
143872023-12-10T18:55:29.054Z 26 520-539 114 114 114 202 202 202 F F F
143882023-12-10T18:55:29.054Z 27 540-559 116 114 116 204 202 204 F F F <---
143892023-12-10T18:55:29.054Z 28 560-579 114 114 114 202 202 202 F F F
143902023-12-10T18:55:29.054Z 29 580-599 113 113 113 201 201 201 F F F
143912023-12-10T18:55:29.054ZMax gen: 116, Max flush: 204
143922023-12-10T18:55:29.054ZError: Difference in extent metadata found!
143932023-12-10T18:55:29.054ZA Difference in extent metadata is expected here
143942023-12-10T18:55:29.054ZOn loop 58
143952023-12-10T18:55:29.054Z
143962023-12-10T18:55:29.054ZVerifying data now
143972023-12-10T18:55:33.952ZNo change after verify
143982023-12-10T18:55:33.953ZLoop: 58 Downstairs dump after verify (and repair):
143992023-12-10T18:55:33.970ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
144002023-12-10T18:55:33.970Z 0 000-019 117 117 117 205 205 205 F F F
144012023-12-10T18:55:33.983Z 1 020-039 116 116 116 203 203 203 F F F
144022023-12-10T18:55:33.984Z 2 040-059 108 108 108 190 190 190 F F F
144032023-12-10T18:55:33.984Z 3 060-079 117 117 117 205 205 205 F F F
144042023-12-10T18:55:33.987Z 4 080-099 116 116 116 203 203 203 F F F
144052023-12-10T18:55:33.987Z 5 100-119 112 112 112 198 198 198 F F F
144062023-12-10T18:55:33.988Z 6 120-139 108 108 108 190 190 190 F F F
144072023-12-10T18:55:33.988Z 7 140-159 112 112 112 199 199 199 F F F
144082023-12-10T18:55:33.988Z 8 160-179 116 116 116 204 204 204 F F F
144092023-12-10T18:55:33.988Z 9 180-199 114 114 114 202 202 202 F F F
144102023-12-10T18:55:33.988Z 10 200-219 112 112 112 197 197 197 F F F
144112023-12-10T18:55:33.988Z 11 220-239 112 112 112 199 199 199 F F F
144122023-12-10T18:55:33.988Z 12 240-259 114 114 114 202 202 202 F F F
144132023-12-10T18:55:33.989Z 13 260-279 117 117 117 205 205 205 F F F
144142023-12-10T18:55:33.989Z 14 280-299 117 117 117 205 205 205 F F F
144152023-12-10T18:55:33.989Z 15 300-319 117 117 117 205 205 205 F F F
144162023-12-10T18:55:33.989Z 16 320-339 114 114 114 202 202 202 F F F
144172023-12-10T18:55:33.990Z 17 340-359 117 117 117 205 205 205 F F F
144182023-12-10T18:55:33.990Z 18 360-379 114 114 114 202 202 202 F F F
144192023-12-10T18:55:33.990Z 19 380-399 117 117 117 205 205 205 F F F
144202023-12-10T18:55:33.990Z 20 400-419 112 112 112 198 198 198 F F F
144212023-12-10T18:55:33.991Z 21 420-439 108 108 108 189 189 189 F F F
144222023-12-10T18:55:33.991Z 22 440-459 116 116 116 204 204 204 F F F
144232023-12-10T18:55:33.991Z 23 460-479 114 114 114 202 202 202 F F F
144242023-12-10T18:55:33.991Z 24 480-499 111 111 111 196 196 196 F F F
144252023-12-10T18:55:33.992Z 25 500-519 116 116 116 203 203 203 F F F
144262023-12-10T18:55:33.992Z 26 520-539 114 114 114 202 202 202 F F F
144272023-12-10T18:55:33.992Z 27 540-559 116 116 116 204 204 204 F F F
144282023-12-10T18:55:33.993Z 28 560-579 114 114 114 202 202 202 F F F
144292023-12-10T18:55:33.994Z 29 580-599 113 113 113 201 201 201 F F F
144302023-12-10T18:55:33.994ZMax gen: 117, Max flush: 205
144312023-12-10T18:55:33.994Z
144322023-12-10T18:55:33.994ZBegin loop 59
144332023-12-10T18:55:33.994ZDownstairs to restart: 0
144342023-12-10T18:55:35.102Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.010304898Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634}
144352023-12-10T18:55:35.102Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.010730754Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634}
144362023-12-10T18:55:35.102Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.010782845Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634}
144372023-12-10T18:55:35.102Z{"msg":"Crucible 6af0c068-ed02-4f03-bd18-7462a7ca8923 has session id: cc12f735-dbc5-49cd-b9cb-9160481d224e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.010846513Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144382023-12-10T18:55:35.153Z{"msg":"Crucible runtime is spawned
144392023-12-10T18:55:35.154ZUpstairs opts: Upstairs UUID: 6af0c068-ed02-4f03-bd18-7462a7ca8923, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.011018115Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144402023-12-10T18:55:35.154Z{"msg":"Crucible stats registered with UUID: 6af0c068-ed02-4f03-bd18-7462a7ca8923","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.011071981Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144412023-12-10T18:55:35.154Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.011154948Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e","task":"up_listen"}
144422023-12-10T18:55:35.158Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.011184752Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144432023-12-10T18:55:35.158Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.011205513Z","hostname":"The guest has requested activation with gen:118
144442023-12-10T18:55:35.159Zip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144452023-12-10T18:55:35.159Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.011514357Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"looper":"2","session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144462023-12-10T18:55:35.159Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.011593196Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"looper":"0","session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144472023-12-10T18:55:35.159Z{"msg":"[2] 6af0c068-ed02-4f03-bd18-7462a7ca8923 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.011649402Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"looper":"2","session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144482023-12-10T18:55:35.160Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.011720226Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"looper":"1","session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144492023-12-10T18:55:35.160Z{"msg":"[0] 6af0c068-ed02-4f03-bd18-7462a7ca8923 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.011768268Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"looper":"0","session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144502023-12-10T18:55:35.160Z{"msg":"[1] 6af0c068-ed02-4f03-bd18-7462a7ca8923 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.011841046Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"looper":"1","session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144512023-12-10T18:55:35.160Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.012134229Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144522023-12-10T18:55:35.160Z{"msg":"6af0c068-ed02-4f03-bd18-7462a7ca8923 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.012191034Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144532023-12-10T18:55:35.161Z{"msg":"Set desired generation to :118","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.012216744Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144542023-12-10T18:55:35.162Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.012476887Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144552023-12-10T18:55:35.162Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.012571361Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144562023-12-10T18:55:35.163Z{"msg":"[2] 6af0c068-ed02-4f03-bd18-7462a7ca8923 (cc12f735-dbc5-49cd-b9cb-9160481d224e) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.012809166Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144572023-12-10T18:55:35.163Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.012871055Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144582023-12-10T18:55:35.163Z{"msg":"[2] client is_active_req TRUE, promote! session cc12f735-dbc5-49cd-b9cb-9160481d224e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.012896413Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144592023-12-10T18:55:35.164Z{"msg":"[1] 6af0c068-ed02-4f03-bd18-7462a7ca8923 (cc12f735-dbc5-49cd-b9cb-9160481d224e) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.013084834Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144602023-12-10T18:55:35.164Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.013151366Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144612023-12-10T18:55:35.164Z{"msg":"[1] client is_active_req TRUE, promote! session cc12f735-dbc5-49cd-b9cb-9160481d224e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.013196255Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144622023-12-10T18:55:35.165Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.0136148Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144632023-12-10T18:55:35.171Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.013742342Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144642023-12-10T18:55:35.172Z{"msg":"6af0c068-ed02-4f03-bd18-7462a7ca8923 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.013830967Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144652023-12-10T18:55:35.172Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.013887736Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144662023-12-10T18:55:35.172Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.013918502Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144672023-12-10T18:55:35.172Z{"msg":"6af0c068-ed02-4f03-bd18-7462a7ca8923 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.013981397Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144682023-12-10T18:55:35.172Z{"msg":"[2] 6af0c068-ed02-4f03-bd18-7462a7ca8923 (cc12f735-dbc5-49cd-b9cb-9160481d224e) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.014410755Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144692023-12-10T18:55:35.173Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.014518264Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144702023-12-10T18:55:35.173Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:34.014576162Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144712023-12-10T18:55:35.173Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.014633415Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144722023-12-10T18:55:35.173Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.01469053Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144732023-12-10T18:55:35.173Z{"msg":"6af0c068-ed02-4f03-bd18-7462a7ca8923 New WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.014748417Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144742023-12-10T18:55:35.174Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.014804904Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144752023-12-10T18:55:35.174Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.014865007Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144762023-12-10T18:55:35.174Z{"msg":"[1] 6af0c068-ed02-4f03-bd18-7462a7ca8923 (cc12f735-dbc5-49cd-b9cb-9160481d224e) New WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.014925288Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144772023-12-10T18:55:35.174Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.014980914Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144782023-12-10T18:55:35.174Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:34.015036966Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144792023-12-10T18:55:35.174Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.015092564Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144802023-12-10T18:55:35.175Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.015155346Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144812023-12-10T18:55:35.175Z{"msg":"6af0c068-ed02-4f03-bd18-7462a7ca8923 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.015195323Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144822023-12-10T18:55:35.175Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.015229265Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144832023-12-10T18:55:35.175Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.015264161Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144842023-12-10T18:55:35.176Z{"msg":"[0] 6af0c068-ed02-4f03-bd18-7462a7ca8923 (cc12f735-dbc5-49cd-b9cb-9160481d224e) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.015329299Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144852023-12-10T18:55:35.176Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.015386446Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144862023-12-10T18:55:35.176Z{"msg":"[0] client is_active_req TRUE, promote! session cc12f735-dbc5-49cd-b9cb-9160481d224e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.015444377Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144872023-12-10T18:55:35.176Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.015546812Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144882023-12-10T18:55:35.177Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.015613819Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144892023-12-10T18:55:35.177Z{"msg":"6af0c068-ed02-4f03-bd18-7462a7ca8923 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.015660731Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144902023-12-10T18:55:35.177Z{"msg":"[0] 6af0c068-ed02-4f03-bd18-7462a7ca8923 (cc12f735-dbc5-49cd-b9cb-9160481d224e) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.01616433Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144912023-12-10T18:55:35.178Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.016237981Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144922023-12-10T18:55:35.178Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:34.016283219Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144932023-12-10T18:55:35.179Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.016306218Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144942023-12-10T18:55:35.179Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.016328947Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144952023-12-10T18:55:35.179Z{"msg":"6af0c068-ed02-4f03-bd18-7462a7ca8923 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.016370393Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144962023-12-10T18:55:35.179Z{"msg":"[0]R flush_numbers[0..12]: [205, 203, 190, 205, 203, 198, 190, 199, 204, 202, 197, 199]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.01641491Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144972023-12-10T18:55:35.180Z{"msg":"[0]R generation[0..12]: [117, 116, 108, 117, 116, 112, 108, 112, 116, 114, 112, 112]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.016469471Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144982023-12-10T18:55:35.180Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.016532546Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
144992023-12-10T18:55:35.180Z{"msg":"[1]R flush_numbers[0..12]: [205, 203, 190, 205, 203, 198, 190, 199, 204, 202, 197, 199]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:118
145002023-12-10T18:55:35.180ZWait for a query_work_queue command to finish before sending IO
145012023-12-10T18:55:35.181Z,"time":"2023-12-10T18:55:34.016610064Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
145022023-12-10T18:55:35.181Z{"msg":"[1]R generation[0..12]: [117, 116, 108, 117, 116, 112, 108, 112, 116, 114, 112, 112]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.016832494Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
145032023-12-10T18:55:35.181Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.016902491Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
145042023-12-10T18:55:35.181Z{"msg":"[2]R flush_numbers[0..12]: [205, 203, 190, 205, 203, 198, 190, 199, 204, 202, 197, 199]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.016948386Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
145052023-12-10T18:55:35.182Z{"msg":"[2]R generation[0..12]: [117, 116, 108, 117, 116, 112, 108, 112, 116, 114, 112, 112]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.016996527Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
145062023-12-10T18:55:35.182Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.017048772Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
145072023-12-10T18:55:35.182Z{"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.01710015Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
145082023-12-10T18:55:35.182Z{"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.017144585Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
145092023-12-10T18:55:35.182Z{"msg":"Next flush: 206","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.017186274Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
145102023-12-10T18:55:35.182Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.017236451Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
145112023-12-10T18:55:35.182Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.017288576Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
145122023-12-10T18:55:35.183Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.017339901Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
145132023-12-10T18:55:35.183Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.01738277Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
145142023-12-10T18:55:35.183Z{"msg":"6af0c068-ed02-4f03-bd18-7462a7ca8923 is now active with session: cc12f735-dbc5-49cd-b9cb-9160481d224e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.017426951Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
145152023-12-10T18:55:35.184Z{"msg":"6af0c068-ed02-4f03-bd18-7462a7ca8923 Set Active after no repair","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
145162023-12-10T18:55:35.184Z,"time":"2023-12-10T18:55:34.017488926Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
145172023-12-10T18:55:35.185Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.017598717Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
145182023-12-10T18:55:35.185Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.017645914Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
145192023-12-10T18:55:35.185Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.017688646Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
145202023-12-10T18:55:35.185Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.017730544Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
145212023-12-10T18:55:35.186Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.017772491Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
145222023-12-10T18:55:35.186Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.017814574Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
145232023-12-10T18:55:35.186Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.017856055Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
145242023-12-10T18:55:35.186Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.017898329Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
145252023-12-10T18:55:35.186Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.017940144Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
145262023-12-10T18:55:35.186Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.017981512Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
145272023-12-10T18:55:35.186Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:34.018022263Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1634,"session_id":"cc12f735-dbc5-49cd-b9cb-9160481d224e"}
145282023-12-10T18:55:35.186ZLoading write count information from file "/tmp/test_repair_verify.data"
145292023-12-10T18:55:35.187ZRun Repair workload
145302023-12-10T18:55:35.187Z01/30 Write block 459 len 20480 data: 8 7 6 8 8
145312023-12-10T18:55:35.187Z02/30 Write block 227 len 20480 data: 14 13 12 16 15
145322023-12-10T18:55:35.187Z03/30 Write block 262 len 8192 data: 9 11
145332023-12-10T18:55:35.187Z04/30 Read block 405 len 20480
145342023-12-10T18:55:35.187Z05/30 Flush
145352023-12-10T18:55:35.188ZError: Difference in extent metadata found!
145362023-12-10T18:55:35.188Z06/30 Write block 171 len 4096 data: 11
145372023-12-10T18:55:35.188Z07/30 Write block 125 len 16384 data: 13 13 12 13
145382023-12-10T18:55:35.209Z08/30 Read block 491 len 12288
145392023-12-10T18:55:35.209Z09/30 Read block 188 len 20480
145402023-12-10T18:55:35.209Z10/30 Read block 245 len 16384
145412023-12-10T18:55:35.209Z11/30 Read block 337 len 16384
145422023-12-10T18:55:35.209Z12/30 Read block 495 len 32768
145432023-12-10T18:55:35.209Z13/30 Read block 348 len 20480
145442023-12-10T18:55:35.209Z14/30 Read block 211 len 8192
145452023-12-10T18:55:35.209Z15/30 Read block 118 len 28672
145462023-12-10T18:55:35.209Z16/30 Read block 477 len 12288
145472023-12-10T18:55:35.209Z17/30 Read block 27 len 16384
145482023-12-10T18:55:35.210Z18/30 Read block 527 len 4096
145492023-12-10T18:55:35.210Z19/30 Read block 57 len 28672
145502023-12-10T18:55:35.210Z20/30 Read block 207 len 8192
145512023-12-10T18:55:35.210Z21/30 Read block 436 len 20480
145522023-12-10T18:55:35.210Z22/30 Flush
145532023-12-10T18:55:35.210Z23/30 Flush
145542023-12-10T18:55:35.210Z24/30 Write block 159 len 28672 data: 9 9 10 15 15 14 13
145552023-12-10T18:55:35.210Z25/30 Read block 500 len 32768
145562023-12-10T18:55:35.210Z26/30 Write block 593 len 12288 data: 9 8 7
145572023-12-10T18:55:35.210Z27/30 Write block 450 len 32768 data: 8 9 10 10 8 9 9 9
145582023-12-10T18:55:35.210Z28/30 Read block 56 len 4096
145592023-12-10T18:55:35.211Z29/30 Read block 399 len 24576
145602023-12-10T18:55:35.211Z30/30 Write block 22 len 20480 data: 8 7 8 7 9
145612023-12-10T18:55:35.211Z----------------------------------------------------------------
145622023-12-10T18:55:35.211Z Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:25
145632023-12-10T18:55:35.211ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
145642023-12-10T18:55:35.211Z 6 Acked 1005 Write 1 Done Done Done false
145652023-12-10T18:55:35.211Z 7 Acked 1006 Write 4 Done Done Done false
145662023-12-10T18:55:35.211Z 8 Acked 1007 Read 3 Done Done Done false
145672023-12-10T18:55:35.211Z 9 Acked 1008 Read 5 Done Done Done false
145682023-12-10T18:55:35.211Z 10 Acked 1009 Read 4 Done Done Done false
145692023-12-10T18:55:35.211Z 11 Acked 1010 Read 4 Done Done Done false
145702023-12-10T18:55:35.211Z 12 Acked 1011 Read 8 Done Done Done false
145712023-12-10T18:55:35.211Z 13 Acked 1012 Read 5 Sent Done Done false
145722023-12-10T18:55:35.212Z 14 Acked 1013 Read 2 Sent Done Done false
145732023-12-10T18:55:35.212Z 15 Acked 1014 Read 7 Sent Done Done false
145742023-12-10T18:55:35.213Z 16 Acked 1015 Read 3 Sent Done Done false
145752023-12-10T18:55:35.213Z 17 Acked 1016 Read 4 Sent Done Done false
145762023-12-10T18:55:35.213Z 18 Acked 1017 Read 1 Sent Done Done false
145772023-12-10T18:55:35.213Z 19 Acked 1018 Read 7 Sent Done Done false
145782023-12-10T18:55:35.213Z 20 Acked 1019 Read 2 Sent Done Done false
145792023-12-10T18:55:35.213Z 21 Acked 1020 Read 5 Sent Done Done false
145802023-12-10T18:55:35.213Z 22 Acked 1021 Flush 0 Sent Done Done false
145812023-12-10T18:55:35.213Z 23 Acked 1022 Flush 0 Sent Done Done false
145822023-12-10T18:55:35.213Z 24 Acked 1023 Write 7 Sent Done Done false
145832023-12-10T18:55:35.213Z 25 Acked 1024 Read 8 Sent Done Done false
145842023-12-10T18:55:35.213Z 26 Acked 1025 Write 3 Sent Done Done false
145852023-12-10T18:55:35.213Z 27 Acked 1026 Write 8 Sent Done Done false
145862023-12-10T18:55:35.214Z 28 Acked 1027 Read 1 Sent Done Done false
145872023-12-10T18:55:35.214Z 29 Acked 1028 Read 6 Sent Done Done false
145882023-12-10T18:55:35.214Z 30 Acked 1029 Write 5 Sent Sent Sent false
145892023-12-10T18:55:35.214Z STATES DS:0 DS:1 DS:2 TOTAL
145902023-12-10T18:55:35.214Z New 0 0 0 0
145912023-12-10T18:55:35.214Z Sent 18 1 1 20
145922023-12-10T18:55:35.214Z Done 7 24 24 55
145932023-12-10T18:55:35.214Z Skipped 0 0 0 0
145942023-12-10T18:55:35.214Z Error 0 0 0 0
145952023-12-10T18:55:35.214ZLast Flush: 1004 1022 1022
145962023-12-10T18:55:35.215ZDownstairs last five completed: 1004 1003 1002 1001 1000
145972023-12-10T18:55:35.215ZUpstairs last five completed: 30 29 28 27 26 25
145982023-12-10T18:55:35.223ZWrote out file "/tmp/test_repair_verify.data"
145992023-12-10T18:55:35.223Z
146002023-12-10T18:55:35.223ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
146012023-12-10T18:55:35.224ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
146022023-12-10T18:55:35.231Z 0 000-019 117 117 117 205 205 205 F F F
146032023-12-10T18:55:35.231Z 1 020-039 116 116 116 203 203 203 F T T <---
146042023-12-10T18:55:35.231Z 2 040-059 108 108 108 190 190 190 F F F
146052023-12-10T18:55:35.231Z 3 060-079 117 117 117 205 205 205 F F F
146062023-12-10T18:55:35.231Z 4 080-099 116 116 116 203 203 203 F F F
146072023-12-10T18:55:35.231Z 5 100-119 112 112 112 198 198 198 F F F
146082023-12-10T18:55:35.232Z 6 120-139 108 118 118 190 207 207 T F F <---
146092023-12-10T18:55:35.232Z 7 140-159 112 112 112 199 199 199 F T T <---
146102023-12-10T18:55:35.232Z 8 160-179 116 118 118 204 207 207 T T T <---
146112023-12-10T18:55:35.232Z 9 180-199 114 114 114 202 202 202 F F F
146122023-12-10T18:55:35.232Z 10 200-219 112 112 112 197 197 197 F F F
146132023-12-10T18:55:35.232Z 11 220-239 118 118 118 206 206 206 F F F
146142023-12-10T18:55:35.232Z 12 240-259 114 114 114 202 202 202 F F F
146152023-12-10T18:55:35.233Z 13 260-279 118 118 118 206 206 206 F F F
146162023-12-10T18:55:35.233Z 14 280-299 117 117 117 205 205 205 F F F
146172023-12-10T18:55:35.233Z 15 300-319 117 117 117 205 205 205 F F F
146182023-12-10T18:55:35.233Z 16 320-339 114 114 114 202 202 202 F F F
146192023-12-10T18:55:35.233Z 17 340-359 117 117 117 205 205 205 F F F
146202023-12-10T18:55:35.233Z 18 360-379 114 114 114 202 202 202 F F F
146212023-12-10T18:55:35.233Z 19 380-399 117 117 117 205 205 205 F F F
146222023-12-10T18:55:35.235Z 20 400-419 112 112 112 198 198 198 F F F
146232023-12-10T18:55:35.235Z 21 420-439 108 108 108 189 189 189 F F F
146242023-12-10T18:55:35.235Z 22 440-459 118 118 118 206 206 206 F T T <---
146252023-12-10T18:55:35.235Z 23 460-479 118 118 118 206 206 206 F F F
146262023-12-10T18:55:35.235Z 24 480-499 111 111 111 196 196 196 F F F
146272023-12-10T18:55:35.235Z 25 500-519 116 116 116 203 203 203 F F F
146282023-12-10T18:55:35.236Z 26 520-539 114 114 114 202 202 202 F F F
146292023-12-10T18:55:35.236Z 27 540-559 116 116 116 204 204 204 F F F
146302023-12-10T18:55:35.236Z 28 560-579 114 114 114 202 202 202 F F F
146312023-12-10T18:55:35.236Z 29 580-599 113 113 113 201 201 201 F T T <---
146322023-12-10T18:55:35.236ZMax gen: 118, Max flush: 207
146332023-12-10T18:55:35.236ZA Difference in extent metadata is expected here
146342023-12-10T18:55:35.236ZOn loop 59
146352023-12-10T18:55:35.237Z
146362023-12-10T18:55:35.237ZVerifying data now
146372023-12-10T18:55:38.843ZNo change after verify
146382023-12-10T18:55:38.843ZLoop: 59 Downstairs dump after verify (and repair):
146392023-12-10T18:55:38.860ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
146402023-12-10T18:55:38.860Z 0 000-019 117 117 117 205 205 205 F F F
146412023-12-10T18:55:39.340Z 1 020-039 119 119 119 208 208 208 F F F
146422023-12-10T18:55:39.340Z 2 040-059 108 108 108 190 190 190 F F F
146432023-12-10T18:55:39.340Z 3 060-079 117 117 117 205 205 205 F F F
146442023-12-10T18:55:39.340Z 4 080-099 116 116 116 203 203 203 F F F
146452023-12-10T18:55:39.341Z 5 100-119 112 112 112 198 198 198 F F F
146462023-12-10T18:55:39.341Z 6 120-139 118 118 118 207 207 207 F F F
146472023-12-10T18:55:39.341Z 7 140-159 119 119 119 208 208 208 F F F
146482023-12-10T18:55:39.341Z 8 160-179 119 119 119 208 208 208 F F F
146492023-12-10T18:55:39.342Z 9 180-199 114 114 114 202 202 202 F F F
146502023-12-10T18:55:39.342Z 10 200-219 112 112 112 197 197 197 F F F
146512023-12-10T18:55:39.342Z 11 220-239 118 118 118 206 206 206 F F F
146522023-12-10T18:55:39.343Z 12 240-259 114 114 114 202 202 202 F F F
146532023-12-10T18:55:39.343Z 13 260-279 118 118 118 206 206 206 F F F
146542023-12-10T18:55:39.343Z 14 280-299 117 117 117 205 205 205 F F F
146552023-12-10T18:55:39.343Z 15 300-319 117 117 117 205 205 205 F F F
146562023-12-10T18:55:39.343Z 16 320-339 114 114 114 202 202 202 F F F
146572023-12-10T18:55:39.344Z 17 340-359 117 117 117 205 205 205 F F F
146582023-12-10T18:55:39.344Z 18 360-379 114 114 114 202 202 202 F F F
146592023-12-10T18:55:39.344Z 19 380-399 117 117 117 205 205 205 F F F
146602023-12-10T18:55:39.344Z 20 400-419 112 112 112 198 198 198 F F F
146612023-12-10T18:55:39.344Z 21 420-439 108 108 108 189 189 189 F F F
146622023-12-10T18:55:39.345Z 22 440-459 119 119 119 208 208 208 F F F
146632023-12-10T18:55:39.345Z 23 460-479 118 118 118 206 206 206 F F F
146642023-12-10T18:55:39.345Z 24 480-499 111 111 111 196 196 196 F F F
146652023-12-10T18:55:39.345Z 25 500-519 116 116 116 203 203 203 F F F
146662023-12-10T18:55:39.345Z 26 520-539 114 114 114 202 202 202 F F F
146672023-12-10T18:55:39.345Z 27 540-559 116 116 116 204 204 204 F F F
146682023-12-10T18:55:39.346Z 28 560-579 114 114 114 202 202 202 F F F
146692023-12-10T18:55:39.346Z 29 580-599 119 119 119 208 208 208 F F F
146702023-12-10T18:55:39.346ZMax gen: 119, Max flush: 208
146712023-12-10T18:55:39.346Z
146722023-12-10T18:55:39.346ZBegin loop 60
146732023-12-10T18:55:39.347ZDownstairs to restart: 2
146742023-12-10T18:55:39.349Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.899595256Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642}
146752023-12-10T18:55:39.349Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.899979493Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642}
146762023-12-10T18:55:39.349Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.900044208Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642}
146772023-12-10T18:55:39.350Z{"msg":"Crucible 3bd097e7-01f5-4019-a63c-3f727a5e20bb has session id: 82bdda91-8c7d-42bd-a142-2d6408779504","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.900071175Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
146782023-12-10T18:55:39.350Z{"msg":"Upstairs opts: Upstairs UUID: 3bd097e7-01f5-4019-a63c-3f727a5e20bb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.900158246Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
146792023-12-10T18:55:39.350ZCrucible runtime is spawned
146802023-12-10T18:55:39.350Z{"msg":"Crucible stats registered with UUID: 3bd097e7-01f5-4019-a63c-3f727a5e20bb","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.900215139Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
146812023-12-10T18:55:39.350ZThe guest has requested activation with gen:120
146822023-12-10T18:55:39.350Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.900405687Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"looper":"0","session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
146832023-12-10T18:55:39.351Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.900477785Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504","task":"up_listen"}
146842023-12-10T18:55:39.351Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.900506266Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
146852023-12-10T18:55:39.351Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.900534697Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
146862023-12-10T18:55:39.351Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.900882903Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"looper":"2","session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
146872023-12-10T18:55:39.351Z{"msg":"[0] 3bd097e7-01f5-4019-a63c-3f727a5e20bb looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.900950262Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"looper":"0","session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
146882023-12-10T18:55:39.351Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.900988526Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"looper":"1","session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
146892023-12-10T18:55:39.352Z{"msg":"[2] 3bd097e7-01f5-4019-a63c-3f727a5e20bb looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.901055692Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"looper":"2","session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
146902023-12-10T18:55:39.352Z{"msg":"[1] 3bd097e7-01f5-4019-a63c-3f727a5e20bb looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.901099179Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"looper":"1","session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
146912023-12-10T18:55:39.352Z{"msg":"3bd097e7-01f5-4019-a63c-3f727a5e20bb active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.901630854Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
146922023-12-10T18:55:39.352Z{"msg":"Set desired generation to :120","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.901668932Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
146932023-12-10T18:55:39.352Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.901959049Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
146942023-12-10T18:55:39.352Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.902238832Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
146952023-12-10T18:55:39.352Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.902310434Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
146962023-12-10T18:55:39.353Z{"msg":"[0] 3bd097e7-01f5-4019-a63c-3f727a5e20bb (82bdda91-8c7d-42bd-a142-2d6408779504) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.902559594Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
146972023-12-10T18:55:39.354Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.902645588Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
146982023-12-10T18:55:39.354Z{"msg":"[0] client is_active_req TRUE, promote! session 82bdda91-8c7d-42bd-a142-2d6408779504","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.902689111Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
146992023-12-10T18:55:39.354Z{"msg":"[1] 3bd097e7-01f5-4019-a63c-3f727a5e20bb (82bdda91-8c7d-42bd-a142-2d6408779504) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.902728105Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147002023-12-10T18:55:39.354Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.902806111Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147012023-12-10T18:55:39.354Z{"msg":"[1] client is_active_req TRUE, promote! session 82bdda91-8c7d-42bd-a142-2d6408779504","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.902850078Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147022023-12-10T18:55:39.355Z{"msg":"[2] 3bd097e7-01f5-4019-a63c-3f727a5e20bb (82bdda91-8c7d-42bd-a142-2d6408779504) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.902921496Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147032023-12-10T18:55:39.355Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.902947361Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147042023-12-10T18:55:39.355Z{"msg":"[2] client is_active_req TRUE, promote! session 82bdda91-8c7d-42bd-a142-2d6408779504","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.902977916Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147052023-12-10T18:55:39.355Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.903377101Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147062023-12-10T18:55:39.355Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.903445594Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147072023-12-10T18:55:39.356Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.90347568Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147082023-12-10T18:55:39.356Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.903501955Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147092023-12-10T18:55:39.356Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.903578969Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147102023-12-10T18:55:39.356Z{"msg":"3bd097e7-01f5-4019-a63c-3f727a5e20bb WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.903637917Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147112023-12-10T18:55:39.356Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.903666908Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147122023-12-10T18:55:39.356Z{"msg":"3bd097e7-01f5-4019-a63c-3f727a5e20bb WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.903734664Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147132023-12-10T18:55:39.357Z{"msg":"3bd097e7-01f5-4019-a63c-3f727a5e20bb WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.903814088Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147142023-12-10T18:55:39.357Z{"msg":"[1] 3bd097e7-01f5-4019-a63c-3f727a5e20bb (82bdda91-8c7d-42bd-a142-2d6408779504) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.904231528Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147152023-12-10T18:55:39.359Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.904334778Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147162023-12-10T18:55:39.360Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:38.904470458Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147172023-12-10T18:55:39.360Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.904513351Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147182023-12-10T18:55:39.360Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.904556164Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147192023-12-10T18:55:39.360Z{"msg":"3bd097e7-01f5-4019-a63c-3f727a5e20bb WaitActive WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.90466948Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147202023-12-10T18:55:39.360Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.904724161Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147212023-12-10T18:55:39.360Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.90474812Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147222023-12-10T18:55:39.360Z{"msg":"[2] 3bd097e7-01f5-4019-a63c-3f727a5e20bb (82bdda91-8c7d-42bd-a142-2d6408779504) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.904773586Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147232023-12-10T18:55:39.361Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.90479488Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147242023-12-10T18:55:39.361Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:38.904835847Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147252023-12-10T18:55:39.362Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.904858196Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147262023-12-10T18:55:39.362Z{"msg":"[0] 3bd097e7-01f5-4019-a63c-3f727a5e20bb (82bdda91-8c7d-42bd-a142-2d6408779504) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.904904949Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147272023-12-10T18:55:39.362Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.904927555Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147282023-12-10T18:55:39.362ZThe guest has finished waiting for activation with:120
147292023-12-10T18:55:39.363Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO
147302023-12-10T18:55:39.363Zlevel":40,"time":"2023-12-10T18:55:38.904955432Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147312023-12-10T18:55:39.363Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.904976109Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147322023-12-10T18:55:39.363Z{"msg":"3bd097e7-01f5-4019-a63c-3f727a5e20bb WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.904996611Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147332023-12-10T18:55:39.364Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.905017103Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147342023-12-10T18:55:39.364Z{"msg":"[0]R flush_numbers[0..12]: [205, 208, 190, 205, 203, 198, 207, 208, 208, 202, 197, 206]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.905038465Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147352023-12-10T18:55:39.364Z{"msg":"[0]R generation[0..12]: [117, 119, 108, 117, 116, 112, 118, 119, 119, 114, 112, 118]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.905059684Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147362023-12-10T18:55:39.364Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.905140943Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147372023-12-10T18:55:39.364Z{"msg":"[1]R flush_numbers[0..12]: [205, 208, 190, 205, 203, 198, 207, 208, 208, 202, 197, 206]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.905183428Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147382023-12-10T18:55:39.364Z{"msg":"[1]R generation[0..12]: [117, 119, 108, 117, 116, 112, 118, 119, 119, 114, 112, 118]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.905221971Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147392023-12-10T18:55:39.365Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.905289119Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147402023-12-10T18:55:39.365Z{"msg":"[2]R flush_numbers[0..12]: [205, 208, 190, 205, 203, 198, 207, 208, 208, 202, 197, 206]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.905328477Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147412023-12-10T18:55:39.399Z{"msg":"[2]R generation[0..12]: [117, 119, 108, 117, 116, 112, 118, 119, 119, 114, 112, 118]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.905452569Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147422023-12-10T18:55:39.400Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.905495279Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147432023-12-10T18:55:39.400Z{"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.905531874Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147442023-12-10T18:55:39.400Z{"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.90558771Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147452023-12-10T18:55:39.400Z{"msg":"Next flush: 209","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.905678597Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147462023-12-10T18:55:39.401Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.905714651Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147472023-12-10T18:55:39.401Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.905778904Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147482023-12-10T18:55:39.401Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.905803548Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147492023-12-10T18:55:39.401Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.905855194Z","hostname":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
147502023-12-10T18:55:39.401Z"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147512023-12-10T18:55:39.401Z{"msg":"3bd097e7-01f5-4019-a63c-3f727a5e20bb is now active with session: 82bdda91-8c7d-42bd-a142-2d6408779504","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.90593203Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147522023-12-10T18:55:39.402Z{"msg":"3bd097e7-01f5-4019-a63c-3f727a5e20bb Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.90595887Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147532023-12-10T18:55:39.402Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.906000802Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147542023-12-10T18:55:39.402Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.906023319Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147552023-12-10T18:55:39.402Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.906064356Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147562023-12-10T18:55:39.402Z{"msg":"3bd097e7-01f5-4019-a63c-3f727a5e20bb Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.906087425Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147572023-12-10T18:55:39.403Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.906127315Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147582023-12-10T18:55:39.403Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.90614941Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147592023-12-10T18:55:39.403Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.906169671Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147602023-12-10T18:55:39.404Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.90618916Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147612023-12-10T18:55:39.404Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.906208831Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147622023-12-10T18:55:39.404Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.906248632Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147632023-12-10T18:55:39.404Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.906270332Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147642023-12-10T18:55:39.404Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.906310185Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147652023-12-10T18:55:39.404Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.906332058Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147662023-12-10T18:55:39.405Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:38.906371553Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1642,"session_id":"82bdda91-8c7d-42bd-a142-2d6408779504"}
147672023-12-10T18:55:39.405ZLoading write count information from file "/tmp/test_repair_verify.data"
147682023-12-10T18:55:39.405ZRun Repair workload
147692023-12-10T18:55:39.406Z01/30 Write block 65 len 20480 data: 9 10 9 12 12
147702023-12-10T18:55:39.406Z02/30 Read block 233 len 32768
147712023-12-10T18:55:39.406Z03/30 Flush
147722023-12-10T18:55:39.406Z04/30 Write block 264 len 24576 data: 10 11 11 11 12 12
147732023-12-10T18:55:39.406Z05/30 Write block 469 len 8192 data: 12 11
147742023-12-10T18:55:39.407Z06/30 Write block 561 len 20480 data: 9 11 9 10 10
147752023-12-10T18:55:39.407Z07/30 Write block 79 len 4096 data: 13
147762023-12-10T18:55:39.407Z08/30 Read block 322 len 24576
147772023-12-10T18:55:39.407Z09/30 Read block 251 len 4096
147782023-12-10T18:55:39.407Z10/30 Read block 563 len 36864
147792023-12-10T18:55:39.408Z11/30 Read block 328 len 12288
147802023-12-10T18:55:39.408Z12/30 Read block 226 len 36864
147812023-12-10T18:55:39.408Z13/30 Read block 237 len 12288
147822023-12-10T18:55:39.408Z14/30 Read block 8 len 4096
147832023-12-10T18:55:39.409Z15/30 Read block 567 len 12288
147842023-12-10T18:55:39.409Z16/30 Flush
147852023-12-10T18:55:39.409Z17/30 Flush
147862023-12-10T18:55:39.409Z18/30 Write block 139 len 8192 data: 6 5
147872023-12-10T18:55:39.409Z19/30 Read block 532 len 20480
147882023-12-10T18:55:39.409Z20/30 Flush
147892023-12-10T18:55:39.409Z21/30 Write block 537 len 4096 data: 8
147902023-12-10T18:55:39.410Z22/30 Read block 586 len 36864
147912023-12-10T18:55:39.410Z23/30 Write block 366 len 32768 data: 6 6 6 7 6 7 8 9
147922023-12-10T18:55:39.410Z24/30 Read block 219 len 24576
147932023-12-10T18:55:39.410Z25/30 Write block 112 len 28672 data: 9 10 8 10 11 10 10
147942023-12-10T18:55:39.410Z26/30 Write block 115 len 24576 data: 11 12 11 11 11 12
147952023-12-10T18:55:39.410Z27/30 Write block 78 len 28672 data: 11 14 12 13 13 13 13
147962023-12-10T18:55:39.411Z28/30 Write block 77 len 8192 data: 13 12
147972023-12-10T18:55:39.411Z29/30 Read block 296 len 20480
147982023-12-10T18:55:39.411Z30/30 Read block 63 len 36864
147992023-12-10T18:55:39.411Z----------------------------------------------------------------
148002023-12-10T18:55:39.411Z Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:27
148012023-12-10T18:55:39.412ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
148022023-12-10T18:55:39.412Z 4 Acked 1003 Write 6 Done Done Done false
148032023-12-10T18:55:39.412Z 5 Acked 1004 Write 2 Done Done Done false
148042023-12-10T18:55:39.412Z 6 Acked 1005 Write 5 Done Done Done false
148052023-12-10T18:55:39.412Z 7 Acked 1006 Write 1 Done Done Done false
148062023-12-10T18:55:39.412Z 8 Acked 1007 Read 6 Done Done Done false
148072023-12-10T18:55:39.412Z 9 Acked 1008 Read 1 Done Done Done false
148082023-12-10T18:55:39.412Z 10 Acked 1009 Read 9 Done Done Done false
148092023-12-10T18:55:39.412Z 11 Acked 1010 Read 3 Done Done Done false
148102023-12-10T18:55:39.413Z 12 Acked 1011 Read 9 Done Done Done false
148112023-12-10T18:55:39.413Z 13 Acked 1012 Read 3 Done Done Done false
148122023-12-10T18:55:39.413Z 14 Acked 1013 Read 1 Done Done Sent false
148132023-12-10T18:55:39.413Z 15 Acked 1014 Read 3 Done Done Sent false
148142023-12-10T18:55:39.413Z 16 Acked 1015 Flush 0 Done Done Sent false
148152023-12-10T18:55:39.413Z 17 Acked 1016 Flush 0 Done Done Sent false
148162023-12-10T18:55:39.414Z 18 Acked 1017 Write 2 Done Done Sent false
148172023-12-10T18:55:39.414Z 19 Acked 1018 Read 5 Done Done Sent false
148182023-12-10T18:55:39.414Z 20 Acked 1019 Flush 0 Done Done Sent false
148192023-12-10T18:55:39.415Z 21 Acked 1020 Write 1 Done Done Sent false
148202023-12-10T18:55:39.416Z 22 Acked 1021 Read 9 Done Done Sent false
148212023-12-10T18:55:39.416Z 23 Acked 1022 Write 8 Done Done Sent false
148222023-12-10T18:55:39.416Z 24 Acked 1023 Read 6 Done Done Sent false
148232023-12-10T18:55:39.416Z 25 Acked 1024 Write 7 Done Done Sent false
148242023-12-10T18:55:39.416Z 26 Acked 1025 Write 6 Done Done Sent false
148252023-12-10T18:55:39.416Z 27 Acked 1026 Write 7 Done Done Sent false
148262023-12-10T18:55:39.417Z 28 Acked 1027 Write 2 Done Done Sent false
148272023-12-10T18:55:39.417Z 29 Acked 1028 Read 5 Done Done Sent false
148282023-12-10T18:55:39.417Z 30 Acked 1029 Read 9 Done Done Sent false
148292023-12-10T18:55:39.417Z STATES DS:0 DS:1 DS:2 TOTAL
148302023-12-10T18:55:39.417Z New 0 0 0 0
148312023-12-10T18:55:39.417Z Sent 0 0 17 17
148322023-12-10T18:55:39.417Z Done 27 27 10 64
148332023-12-10T18:55:39.417Z Skipped 0 0 0 0
148342023-12-10T18:55:39.418Z Error 0 0 0 0
148352023-12-10T18:55:39.418ZLast Flush: 1019 1019 1002
148362023-12-10T18:55:39.418ZDownstairs last five completed: 1002 1001 1000
148372023-12-10T18:55:39.418ZUpstairs last five completed: 30 29 28 27 26 25
148382023-12-10T18:55:39.418ZWrote out file "/tmp/test_repair_verify.data"
148392023-12-10T18:55:39.418Z
148402023-12-10T18:55:39.418ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
148412023-12-10T18:55:39.434ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
148422023-12-10T18:55:39.434Z 0 000-019 117 117 117 205 205 205 F F F
148432023-12-10T18:55:39.435Z 1 020-039 119 119 119 208 208 208 F F F
148442023-12-10T18:55:39.435Z 2 040-059 108 108 108 190 190 190 F F F
148452023-12-10T18:55:39.435Z 3 060-079 120 120 120 210 210 209 T T T <---
148462023-12-10T18:55:39.436Z 4 080-099 116 116 116 203 203 203 T T F <---
148472023-12-10T18:55:39.436Z 5 100-119 112 112 112 198 198 198 T T F <---
148482023-12-10T18:55:39.436Z 6 120-139 120 120 118 212 212 207 T T F <---
148492023-12-10T18:55:39.436Z 7 140-159 120 120 119 212 212 208 F F F <---
148502023-12-10T18:55:39.437Z 8 160-179 119 119 119 208 208 208 F F F
148512023-12-10T18:55:39.437Z 9 180-199 114 114 114 202 202 202 F F F
148522023-12-10T18:55:39.438Z 10 200-219 112 112 112 197 197 197 F F F
148532023-12-10T18:55:39.438Z 11 220-239 118 118 118 206 206 206 F F F
148542023-12-10T18:55:39.438Z 12 240-259 114 114 114 202 202 202 F F F
148552023-12-10T18:55:39.438Z 13 260-279 120 120 118 210 210 206 F F T <---
148562023-12-10T18:55:39.438Z 14 280-299 117 117 117 205 205 205 F F F
148572023-12-10T18:55:39.438Z 15 300-319 117 117 117 205 205 205 F F F
148582023-12-10T18:55:39.439Z 16 320-339 114 114 114 202 202 202 F F F
148592023-12-10T18:55:39.439Z 17 340-359 117 117 117 205 205 205 F F F
148602023-12-10T18:55:39.439Z 18 360-379 114 114 114 202 202 202 T T F <---
148612023-12-10T18:55:39.439Z 19 380-399 117 117 117 205 205 205 F F F
148622023-12-10T18:55:39.439Z 20 400-419 112 112 112 198 198 198 F F F
148632023-12-10T18:55:39.439Z 21 420-439 108 108 108 189 189 189 F F F
148642023-12-10T18:55:39.440Z 22 440-459 119 119 119 208 208 208 F F F
148652023-12-10T18:55:39.440Z 23 460-479 120 120 118 210 210 206 F F T <---
148662023-12-10T18:55:39.440Z 24 480-499 111 111 111 196 196 196 F F F
148672023-12-10T18:55:39.441Z 25 500-519 116 116 116 203 203 203 F F F
148682023-12-10T18:55:39.441Z 26 520-539 114 114 114 202 202 202 T T F <---
148692023-12-10T18:55:39.441Z 27 540-559 116 116 116 204 204 204 F F F
148702023-12-10T18:55:39.441Z 28 560-579 120 120 114 210 210 202 F F T <---
148712023-12-10T18:55:39.442Z 29 580-599 119 119 119 208 208 208 F F F
148722023-12-10T18:55:39.442ZMax gen: 120, Max flush: 212
148732023-12-10T18:55:39.442ZError: Difference in extent metadata found!
148742023-12-10T18:55:39.442ZA Difference in extent metadata is expected here
148752023-12-10T18:55:39.443ZOn loop 60
148762023-12-10T18:55:39.443Z
148772023-12-10T18:55:39.443ZVerifying data now
148782023-12-10T18:55:45.189ZNo change after verify
148792023-12-10T18:55:45.190ZLoop: 60 Downstairs dump after verify (and repair):
148802023-12-10T18:55:46.057ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
148812023-12-10T18:55:46.057Z 0 000-019 117 117 117 205 205 205 F F F
148822023-12-10T18:55:46.057Z 1 020-039 119 119 119 208 208 208 F F F
148832023-12-10T18:55:46.057Z 2 040-059 108 108 108 190 190 190 F F F
148842023-12-10T18:55:46.058Z 3 060-079 121 121 121 213 213 213 F F F
148852023-12-10T18:55:46.060Z 4 080-099 121 121 121 213 213 213 F F F
148862023-12-10T18:55:46.061Z 5 100-119 121 121 121 213 213 213 F F F
148872023-12-10T18:55:46.061Z 6 120-139 121 121 121 213 213 213 F F F
148882023-12-10T18:55:46.061Z 7 140-159 120 120 120 212 212 212 F F F
148892023-12-10T18:55:46.061Z 8 160-179 119 119 119 208 208 208 F F F
148902023-12-10T18:55:46.061Z 9 180-199 114 114 114 202 202 202 F F F
148912023-12-10T18:55:46.061Z 10 200-219 112 112 112 197 197 197 F F F
148922023-12-10T18:55:46.061Z 11 220-239 118 118 118 206 206 206 F F F
148932023-12-10T18:55:46.061Z 12 240-259 114 114 114 202 202 202 F F F
148942023-12-10T18:55:46.061Z 13 260-279 120 120 120 210 210 210 F F F
148952023-12-10T18:55:46.062Z 14 280-299 117 117 117 205 205 205 F F F
148962023-12-10T18:55:46.062Z 15 300-319 117 117 117 205 205 205 F F F
148972023-12-10T18:55:46.062Z 16 320-339 114 114 114 202 202 202 F F F
148982023-12-10T18:55:46.062Z 17 340-359 117 117 117 205 205 205 F F F
148992023-12-10T18:55:46.063Z 18 360-379 121 121 121 213 213 213 F F F
149002023-12-10T18:55:46.063Z 19 380-399 117 117 117 205 205 205 F F F
149012023-12-10T18:55:46.063Z 20 400-419 112 112 112 198 198 198 F F F
149022023-12-10T18:55:46.063Z 21 420-439 108 108 108 189 189 189 F F F
149032023-12-10T18:55:46.063Z 22 440-459 119 119 119 208 208 208 F F F
149042023-12-10T18:55:46.064Z 23 460-479 120 120 120 210 210 210 F F F
149052023-12-10T18:55:46.064Z 24 480-499 111 111 111 196 196 196 F F F
149062023-12-10T18:55:46.064Z 25 500-519 116 116 116 203 203 203 F F F
149072023-12-10T18:55:46.064Z 26 520-539 121 121 121 213 213 213 F F F
149082023-12-10T18:55:46.065Z 27 540-559 116 116 116 204 204 204 F F F
149092023-12-10T18:55:46.065Z 28 560-579 120 120 120 210 210 210 F F F
149102023-12-10T18:55:46.065Z 29 580-599 119 119 119 208 208 208 F F F
149112023-12-10T18:55:46.065ZMax gen: 121, Max flush: 213
149122023-12-10T18:55:46.065Z
149132023-12-10T18:55:46.066ZBegin loop 61
149142023-12-10T18:55:46.066ZDownstairs to restart: 2
149152023-12-10T18:55:46.066Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.095938204Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650}
149162023-12-10T18:55:46.066Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.096365732Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650}
149172023-12-10T18:55:46.066Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.096435791Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650}
149182023-12-10T18:55:46.066Z{"msg":"Crucible 1e07e388-21d0-4f8d-8731-98253e8f9d75 has session id: 723e3d49-744b-4a47-8d94-2c4f17001122","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.096479697Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149192023-12-10T18:55:46.066Z{"msg":"Upstairs opts: Upstairs UUID: 1e07e388-21d0-4f8d-8731-98253e8f9d75, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
149202023-12-10T18:55:46.067Z,"time":"2023-12-10T18:55:45.096532999Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149212023-12-10T18:55:46.067Z{"msg":"Crucible stats registered with UUID: 1e07e388-21d0-4f8d-8731-98253e8f9d75","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.096581434Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149222023-12-10T18:55:46.067ZThe guest has requested activation with gen:122
149232023-12-10T18:55:46.067Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.096829634Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"looper":"1","session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149242023-12-10T18:55:46.067Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.096967838Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"looper":"2","session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149252023-12-10T18:55:46.068Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.09706699Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122","task":"up_listen"}
149262023-12-10T18:55:46.068Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.09711423Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149272023-12-10T18:55:46.068Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.097197282Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149282023-12-10T18:55:46.068Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.097240099Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"looper":"0","session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149292023-12-10T18:55:46.068Z{"msg":"[1] 1e07e388-21d0-4f8d-8731-98253e8f9d75 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.097284059Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"looper":"1","session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149302023-12-10T18:55:46.068Z{"msg":"1e07e388-21d0-4f8d-8731-98253e8f9d75 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.097342049Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149312023-12-10T18:55:46.069Z{"msg":"Set desired generation to :122","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.097366717Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149322023-12-10T18:55:46.069Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.097389493Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149332023-12-10T18:55:46.069Z{"msg":"[0] 1e07e388-21d0-4f8d-8731-98253e8f9d75 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.097503856Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"looper":"0","session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149342023-12-10T18:55:46.069Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.097666126Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149352023-12-10T18:55:46.070Z{"msg":"[2] 1e07e388-21d0-4f8d-8731-98253e8f9d75 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.097784803Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"looper":"2","session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149362023-12-10T18:55:46.070Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.097937355Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149372023-12-10T18:55:46.070Z{"msg":"[1] 1e07e388-21d0-4f8d-8731-98253e8f9d75 (723e3d49-744b-4a47-8d94-2c4f17001122) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.098203811Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149382023-12-10T18:55:46.070Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.098263768Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149392023-12-10T18:55:46.070Z{"msg":"[0] 1e07e388-21d0-4f8d-8731-98253e8f9d75 (723e3d49-744b-4a47-8d94-2c4f17001122) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.098291179Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149402023-12-10T18:55:46.071Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.098312015Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149412023-12-10T18:55:46.071Z{"msg":"[1] client is_active_req TRUE, promote! session 723e3d49-744b-4a47-8d94-2c4f17001122","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.098361346Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149422023-12-10T18:55:46.071Z{"msg":"[0] client is_active_req TRUE, promote! session 723e3d49-744b-4a47-8d94-2c4f17001122","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.098421132Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149432023-12-10T18:55:46.071Z{"msg":"[2] 1e07e388-21d0-4f8d-8731-98253e8f9d75 (723e3d49-744b-4a47-8d94-2c4f17001122) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.099025526Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149442023-12-10T18:55:46.072Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.099064422Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149452023-12-10T18:55:46.072Z{"msg":"[2] client is_active_req TRUE, promote! session 723e3d49-744b-4a47-8d94-2c4f17001122","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.099088059Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149462023-12-10T18:55:46.072Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.099232342Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149472023-12-10T18:55:46.072Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.099292335Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149482023-12-10T18:55:46.073Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.099339558Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149492023-12-10T18:55:46.073Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.099402622Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149502023-12-10T18:55:46.073Z{"msg":"1e07e388-21d0-4f8d-8731-98253e8f9d75 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.099661655Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149512023-12-10T18:55:46.073Z{"msg":"1e07e388-21d0-4f8d-8731-98253e8f9d75 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.099722717Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149522023-12-10T18:55:46.074Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.099926936Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149532023-12-10T18:55:46.074Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.09996214Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149542023-12-10T18:55:46.074Z{"msg":"1e07e388-21d0-4f8d-8731-98253e8f9d75 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.100095773Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149552023-12-10T18:55:46.075Z{"msg":"[0] 1e07e388-21d0-4f8d-8731-98253e8f9d75 (723e3d49-744b-4a47-8d94-2c4f17001122) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.10032111Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149562023-12-10T18:55:46.075Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.100376882Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149572023-12-10T18:55:46.075Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:45.100400788Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149582023-12-10T18:55:46.075Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.100436418Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149592023-12-10T18:55:46.075Z{"msg":"[1] 1e07e388-21d0-4f8d-8731-98253e8f9d75 (723e3d49-744b-4a47-8d94-2c4f17001122) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.100509434Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149602023-12-10T18:55:46.076Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.10057717Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149612023-12-10T18:55:46.076Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:45.100601589Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149622023-12-10T18:55:46.076Z{"msg":"1e07e388-21d0-4f8d-8731-98253e8f9d75 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.100623207Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149632023-12-10T18:55:46.076Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.100667087Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149642023-12-10T18:55:46.077Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.10068902Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149652023-12-10T18:55:46.077Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.100709176Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149662023-12-10T18:55:46.077Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.100728621Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149672023-12-10T18:55:46.078Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.100749127Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149682023-12-10T18:55:46.078Z{"msg":"1e07e388-21d0-4f8d-8731-98253e8f9d75 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.100789483Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149692023-12-10T18:55:46.078Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.100811791Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149702023-12-10T18:55:46.078Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.100831207Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149712023-12-10T18:55:46.079Z{"msg":"[2] 1e07e388-21d0-4f8d-8731-98253e8f9d75 (723e3d49-744b-4a47-8d94-2c4f17001122) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.100873623Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149722023-12-10T18:55:46.079Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.100896107Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149732023-12-10T18:55:46.079Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:45.100917155Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149742023-12-10T18:55:46.079Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.100957087Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149752023-12-10T18:55:46.079Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.100978809Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149762023-12-10T18:55:46.080Z{"msg":"1e07e388-21d0-4f8d-8731-98253e8f9d75 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.100999342Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149772023-12-10T18:55:46.080Z{"msg":"[0]R flush_numbers[0..12]: [205, 208, 190, 213, 213, 213, 213, 212, 208, 202, 197, 206]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.101042061Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149782023-12-10T18:55:46.080Z{"msg":"The guest has finished waiting for activation with:122
149792023-12-10T18:55:46.080Z[0]R generation[0..12]: [117, 119, 108, 121, 121, 121, 121, 120, 119, 114, 112, 118]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
149802023-12-10T18:55:46.531Z,"time":"2023-12-10T18:55:45.101068296Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149812023-12-10T18:55:46.532Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.101111062Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149822023-12-10T18:55:46.532Z{"msg":"[1]R flush_numbers[0..12]: [205, 208, 190, 213, 213, 213, 213, 212, 208, 202, 197, 206]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.101134539Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149832023-12-10T18:55:46.532Z{"msg":"[1]R generation[0..12]: [117, 119, 108, 121, 121, 121, 121, 120, 119, 114, 112, 118]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.101156032Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149842023-12-10T18:55:46.532Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.101202232Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149852023-12-10T18:55:46.533Z{"msg":"[2]R flush_numbers[0..12]: [205, 208, 190, 213, 213, 213, 213, 212, 208, 202, 197, 206]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.101225547Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149862023-12-10T18:55:46.533Z{"msg":"[2]R generation[0..12]: [117, 119, 108, 121, 121, 121, 121, 120, 119, 114, 112, 118]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.101252012Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149872023-12-10T18:55:46.533Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.101290057Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149882023-12-10T18:55:46.533Z{"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.101351592Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149892023-12-10T18:55:46.533Z{"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.101376428Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149902023-12-10T18:55:46.534Z{"msg":"Next flush: 214","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.101396308Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149912023-12-10T18:55:46.534Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.101445142Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149922023-12-10T18:55:46.534Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.101467312Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149932023-12-10T18:55:46.535Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.101487269Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149942023-12-10T18:55:46.535Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.101528194Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149952023-12-10T18:55:46.535Z{"msg":"1e07e388-21d0-4f8d-8731-98253e8f9d75 is now active with session: 723e3d49-744b-4a47-8d94-2c4f17001122","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.101551689Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149962023-12-10T18:55:46.536Z{"msg":"1e07e388-21d0-4f8d-8731-98253e8f9d75 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.101603383Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149972023-12-10T18:55:46.536Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.101625629Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149982023-12-10T18:55:46.537Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.101668775Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
149992023-12-10T18:55:46.544Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.101690355Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
150002023-12-10T18:55:46.544Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.10174179Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
150012023-12-10T18:55:46.545Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.101765977Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
150022023-12-10T18:55:46.545Z{"msg":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
150032023-12-10T18:55:46.545Z"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.101865263Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
150042023-12-10T18:55:46.546Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.101915733Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
150052023-12-10T18:55:46.546Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.101938992Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
150062023-12-10T18:55:46.546Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.101979023Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
150072023-12-10T18:55:46.547Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.102000849Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
150082023-12-10T18:55:46.547Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:45.102041048Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1650,"session_id":"723e3d49-744b-4a47-8d94-2c4f17001122"}
150092023-12-10T18:55:46.547ZLoading write count information from file "/tmp/test_repair_verify.data"
150102023-12-10T18:55:46.548ZRun Repair workload
150112023-12-10T18:55:46.548Z01/30 Write block 347 len 40960 data: 11 10 13 11 12 14 13 11 9 11
150122023-12-10T18:55:46.548Z02/30 Write block 292 len 4096 data: 10
150132023-12-10T18:55:46.548Z03/30 Read block 264 len 20480
150142023-12-10T18:55:46.549Z04/30 Write block 108 len 28672 data: 10 9 10 9 10 11 9
150152023-12-10T18:55:46.549Z05/30 Write block 235 len 12288 data: 11 11 12
150162023-12-10T18:55:46.549Z06/30 Read block 526 len 12288
150172023-12-10T18:55:46.550Z07/30 Read block 189 len 20480
150182023-12-10T18:55:46.550Z08/30 Write block 293 len 8192 data: 10 7
150192023-12-10T18:55:46.550Z09/30 Write block 458 len 4096 data: 10
150202023-12-10T18:55:46.551Z10/30 Read block 226 len 32768
150212023-12-10T18:55:46.551Z11/30 Read block 370 len 24576
150222023-12-10T18:55:46.551Z12/30 Write block 267 len 8192 data: 12 13
150232023-12-10T18:55:46.552Z13/30 Write block 304 len 16384 data: 14 9 10 7
150242023-12-10T18:55:46.552Z14/30 Write block 20 len 8192 data: 10 8
150252023-12-10T18:55:46.552Z15/30 Write block 354 len 32768 data: 12 10 12 10 10 10 8 7
150262023-12-10T18:55:46.552Z16/30 Write block 529 len 4096 data: 8
150272023-12-10T18:55:46.553Z17/30 Write block 442 len 24576 data: 5 7 8 9 8 8
150282023-12-10T18:55:46.553Z18/30 Write block 101 len 24576 data: 7 10 9 6 6 7
150292023-12-10T18:55:46.554Z19/30 Read block 553 len 4096
150302023-12-10T18:55:46.555Z20/30 Flush
150312023-12-10T18:55:46.555Z21/30 Write block 413 len 24576 data: 3 4 6 7 8 9
150322023-12-10T18:55:46.555Z22/30 Read block 51 len 4096
150332023-12-10T18:55:46.556Z23/30 Write block 93 len 36864 data: 11 11 11 8 6 6 7 8 8
150342023-12-10T18:55:46.556Z24/30 Write block 131 len 24576 data: 7 3 4 4 4 4
150352023-12-10T18:55:46.556Z25/30 Read block 474 len 8192
150362023-12-10T18:55:46.557Z26/30 Read block 205 len 40960
150372023-12-10T18:55:46.557Z27/30 Write block 132 len 8192 data: 4 5
150382023-12-10T18:55:46.557Z28/30 Write block 80 len 20480 data: 13 14 14 14 14
150392023-12-10T18:55:46.558Z29/30 Write block 476 len 20480 data: 8 6 4 5 5
150402023-12-10T18:55:46.558Z30/30 Read block 187 len 24576
150412023-12-10T18:55:46.558Z----------------------------------------------------------------
150422023-12-10T18:55:46.559Z Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:30
150432023-12-10T18:55:46.559ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
150442023-12-10T18:55:46.559Z 1 Acked 1000 Write 10 Done Done Sent false
150452023-12-10T18:55:46.559Z 2 Acked 1001 Write 1 Done Done Sent false
150462023-12-10T18:55:46.560Z 3 Acked 1002 Read 5 Done Done Sent false
150472023-12-10T18:55:46.560Z 4 Acked 1003 Write 7 Done Done Sent false
150482023-12-10T18:55:46.560Z 5 Acked 1004 Write 3 Done Done Sent false
150492023-12-10T18:55:46.561Z 6 Acked 1005 Read 3 Done Done Sent false
150502023-12-10T18:55:46.561Z 7 Acked 1006 Read 5 Done Done Sent false
150512023-12-10T18:55:46.561Z 8 Acked 1007 Write 2 Done Done Sent false
150522023-12-10T18:55:46.562Z 9 Acked 1008 Write 1 Done Done Sent false
150532023-12-10T18:55:46.563Z 10 Acked 1009 Read 8 Done Done Sent false
150542023-12-10T18:55:46.564Z 11 Acked 1010 Read 6 Done Done Sent false
150552023-12-10T18:55:46.564Z 12 Acked 1011 Write 2 Done Done Sent false
150562023-12-10T18:55:46.564Z 13 Acked 1012 Write 4 Done Done Sent false
150572023-12-10T18:55:46.565Z 14 Acked 1013 Write 2 Done Done Sent false
150582023-12-10T18:55:46.565Z 15 Acked 1014 Write 8 Done Done Sent false
150592023-12-10T18:55:46.565Z 16 Acked 1015 Write 1 Done Done Sent false
150602023-12-10T18:55:46.566Z 17 Acked 1016 Write 6 Done Done Sent false
150612023-12-10T18:55:46.566Z 18 Acked 1017 Write 6 Done Done Sent false
150622023-12-10T18:55:46.566Z 19 Acked 1018 Read 1 Done Done Sent false
150632023-12-10T18:55:46.566Z 20 Acked 1019 Flush 0 Done Done Sent false
150642023-12-10T18:55:46.567Z 21 Acked 1020 Write 6 Done Done Sent false
150652023-12-10T18:55:46.567Z 22 Acked 1021 Read 1 Done Done Sent false
150662023-12-10T18:55:46.567Z 23 Acked 1022 Write 9 Done Done Sent false
150672023-12-10T18:55:46.568Z 24 Acked 1023 Write 6 Done Done Sent false
150682023-12-10T18:55:46.568Z 25 Acked 1024 Read 2 Done Done Sent false
150692023-12-10T18:55:46.568Z 26 Acked 1025 Read 10 Done Done Sent false
150702023-12-10T18:55:46.568Z 27 Acked 1026 Write 2 Done Done Sent false
150712023-12-10T18:55:46.568Z 28 Acked 1027 Write 5 Done Done Sent false
150722023-12-10T18:55:46.569Z 29 Acked 1028 Write 5 Done Done Sent false
150732023-12-10T18:55:46.569Z 30 Acked 1029 Read 6 Done Done Sent false
150742023-12-10T18:55:46.569Z STATES DS:0 DS:1 DS:2 TOTAL
150752023-12-10T18:55:46.569Z New 0 0 0 0
150762023-12-10T18:55:46.570Z Sent 0 0 30 30
150772023-12-10T18:55:46.570Z Done 30 30 0 60
150782023-12-10T18:55:46.571Z Skipped 0 0 0 0
150792023-12-10T18:55:46.571Z Error 0 0 0 0
150802023-12-10T18:55:47.011ZLast Flush: 1019 1019 0
150812023-12-10T18:55:47.012ZDownstairs last five completed:
150822023-12-10T18:55:47.012ZUpstairs last five completed: 30 29 28 27 26 25
150832023-12-10T18:55:47.012ZWrote out file "/tmp/test_repair_verify.data"
150842023-12-10T18:55:47.013Z
150852023-12-10T18:55:47.013ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
150862023-12-10T18:55:47.013ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
150872023-12-10T18:55:47.013Z 0 000-019 117 117 117 205 205 205 F F F
150882023-12-10T18:55:47.013Z 1 020-039 122 122 119 214 214 208 F F F <---
150892023-12-10T18:55:47.014Z 2 040-059 108 108 108 190 190 190 F F F
150902023-12-10T18:55:47.014Z 3 060-079 121 121 121 213 213 213 F F F
150912023-12-10T18:55:47.014Z 4 080-099 121 121 121 213 213 213 T T F <---
150922023-12-10T18:55:47.014Z 5 100-119 122 122 121 214 214 213 T T F <---
150932023-12-10T18:55:47.014Z 6 120-139 121 121 121 213 213 213 T T F <---
150942023-12-10T18:55:47.014Z 7 140-159 120 120 120 212 212 212 F F F
150952023-12-10T18:55:47.015Z 8 160-179 119 119 119 208 208 208 F F F
150962023-12-10T18:55:47.015Z 9 180-199 114 114 114 202 202 202 F F F
150972023-12-10T18:55:47.015Z 10 200-219 112 112 112 197 197 197 F F F
150982023-12-10T18:55:47.015Z 11 220-239 122 122 118 214 214 206 F F F <---
150992023-12-10T18:55:47.016Z 12 240-259 114 114 114 202 202 202 F F F
151002023-12-10T18:55:47.016Z 13 260-279 122 122 120 214 214 210 F F F <---
151012023-12-10T18:55:47.018Z 14 280-299 122 122 117 214 214 205 F F F <---
151022023-12-10T18:55:47.018Z 15 300-319 122 122 117 214 214 205 F F F <---
151032023-12-10T18:55:47.018Z 16 320-339 114 114 114 202 202 202 F F F
151042023-12-10T18:55:47.018Z 17 340-359 122 122 117 214 214 205 F F F <---
151052023-12-10T18:55:47.018Z 18 360-379 122 122 121 214 214 213 F F F <---
151062023-12-10T18:55:47.018Z 19 380-399 117 117 117 205 205 205 F F F
151072023-12-10T18:55:47.018Z 20 400-419 112 112 112 198 198 198 T T F <---
151082023-12-10T18:55:47.018Z 21 420-439 108 108 108 189 189 189 F F F
151092023-12-10T18:55:47.019Z 22 440-459 122 122 119 214 214 208 F F F <---
151102023-12-10T18:55:47.019Z 23 460-479 120 120 120 210 210 210 T T F <---
151112023-12-10T18:55:47.019Z 24 480-499 111 111 111 196 196 196 T T F <---
151122023-12-10T18:55:47.019Z 25 500-519 116 116 116 203 203 203 F F F
151132023-12-10T18:55:47.020Z 26 520-539 122 122 121 214 214 213 F F F <---
151142023-12-10T18:55:47.020Z 27 540-559 116 116 116 204 204 204 F F F
151152023-12-10T18:55:47.020Z 28 560-579 120 120 120 210 210 210 F F F
151162023-12-10T18:55:47.020Z 29 580-599 119 119 119 208 208 208 F F F
151172023-12-10T18:55:47.020ZMax gen: 122, Max flush: 214
151182023-12-10T18:55:47.021ZError: Difference in extent metadata found!
151192023-12-10T18:55:47.021ZA Difference in extent metadata is expected here
151202023-12-10T18:55:47.021ZOn loop 61
151212023-12-10T18:55:47.022Z
151222023-12-10T18:55:47.022ZVerifying data now
151232023-12-10T18:55:50.597ZNo change after verify
151242023-12-10T18:55:50.597ZLoop: 61 Downstairs dump after verify (and repair):
151252023-12-10T18:55:50.621ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
151262023-12-10T18:55:50.621Z 0 000-019 117 117 117 205 205 205 F F F
151272023-12-10T18:55:50.622Z 1 020-039 122 122 122 214 214 214 F F F
151282023-12-10T18:55:50.622Z 2 040-059 108 108 108 190 190 190 F F F
151292023-12-10T18:55:50.622Z 3 060-079 121 121 121 213 213 213 F F F
151302023-12-10T18:55:50.622Z 4 080-099 123 123 123 215 215 215 F F F
151312023-12-10T18:55:50.622Z 5 100-119 123 123 123 215 215 215 F F F
151322023-12-10T18:55:50.622Z 6 120-139 123 123 123 215 215 215 F F F
151332023-12-10T18:55:50.622Z 7 140-159 120 120 120 212 212 212 F F F
151342023-12-10T18:55:50.640Z 8 160-179 119 119 119 208 208 208 F F F
151352023-12-10T18:55:50.640Z 9 180-199 114 114 114 202 202 202 F F F
151362023-12-10T18:55:50.640Z 10 200-219 112 112 112 197 197 197 F F F
151372023-12-10T18:55:50.640Z 11 220-239 122 122 122 214 214 214 F F F
151382023-12-10T18:55:50.640Z 12 240-259 114 114 114 202 202 202 F F F
151392023-12-10T18:55:50.640Z 13 260-279 122 122 122 214 214 214 F F F
151402023-12-10T18:55:50.640Z 14 280-299 122 122 122 214 214 214 F F F
151412023-12-10T18:55:50.640Z 15 300-319 122 122 122 214 214 214 F F F
151422023-12-10T18:55:50.640Z 16 320-339 114 114 114 202 202 202 F F F
151432023-12-10T18:55:50.641Z 17 340-359 122 122 122 214 214 214 F F F
151442023-12-10T18:55:50.641Z 18 360-379 122 122 122 214 214 214 F F F
151452023-12-10T18:55:50.641Z 19 380-399 117 117 117 205 205 205 F F F
151462023-12-10T18:55:50.641Z 20 400-419 123 123 123 215 215 215 F F F
151472023-12-10T18:55:50.642Z 21 420-439 108 108 108 189 189 189 F F F
151482023-12-10T18:55:50.642Z 22 440-459 122 122 122 214 214 214 F F F
151492023-12-10T18:55:50.642Z 23 460-479 123 123 123 215 215 215 F F F
151502023-12-10T18:55:50.643Z 24 480-499 123 123 123 215 215 215 F F F
151512023-12-10T18:55:50.643Z 25 500-519 116 116 116 203 203 203 F F F
151522023-12-10T18:55:50.643Z 26 520-539 122 122 122 214 214 214 F F F
151532023-12-10T18:55:50.644Z 27 540-559 116 116 116 204 204 204 F F F
151542023-12-10T18:55:50.644Z 28 560-579 120 120 120 210 210 210 F F F
151552023-12-10T18:55:50.644Z 29 580-599 119 119 119 208 208 208 F F F
151562023-12-10T18:55:50.645ZMax gen: 123, Max flush: 215
151572023-12-10T18:55:50.645Z
151582023-12-10T18:55:50.645ZBegin loop 62
151592023-12-10T18:55:50.645ZDownstairs to restart: 2
151602023-12-10T18:55:50.788Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.657601842Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658}
151612023-12-10T18:55:50.788Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.658087848Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658}
151622023-12-10T18:55:50.788Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.658156383Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658}
151632023-12-10T18:55:50.788Z{"msg":"Crucible 9119e673-14cb-443c-80ab-6739285d75eb has session id: a511b1ae-51ec-4f52-a8a9-80702ead4569","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.65819246Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151642023-12-10T18:55:50.788Z{"msg":"Upstairs opts: Upstairs UUID: 9119e673-14cb-443c-80ab-6739285d75eb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.658253425Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151652023-12-10T18:55:50.789Z{"msg":"Crucible stats registered with UUID: 9119e673-14cb-443c-80ab-6739285d75eb","v":0,"name":"crucible","levelCrucible runtime is spawned
151662023-12-10T18:55:50.789Z":30The guest has requested activation with gen:124
151672023-12-10T18:55:50.789Z,"time":"2023-12-10T18:55:50.658398169Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151682023-12-10T18:55:50.789Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.658533275Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569","task":"up_listen"}
151692023-12-10T18:55:50.789Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.658577838Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151702023-12-10T18:55:50.789Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.658754035Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"looper":"2","session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151712023-12-10T18:55:50.789Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.658847018Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151722023-12-10T18:55:50.789Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.658884985Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"looper":"1","session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151732023-12-10T18:55:50.790Z{"msg":"[2] 9119e673-14cb-443c-80ab-6739285d75eb looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.658948508Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"looper":"2","session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151742023-12-10T18:55:50.790Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.658989499Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"looper":"0","session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151752023-12-10T18:55:50.791Z{"msg":"[1] 9119e673-14cb-443c-80ab-6739285d75eb looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.65903101Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"looper":"1","session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151762023-12-10T18:55:50.791Z{"msg":"[0] 9119e673-14cb-443c-80ab-6739285d75eb looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.65907237Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"looper":"0","session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151772023-12-10T18:55:50.791Z{"msg":"9119e673-14cb-443c-80ab-6739285d75eb active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.659385952Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151782023-12-10T18:55:50.791Z{"msg":"Set desired generation to :124","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.659440849Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151792023-12-10T18:55:50.791Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.659553648Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151802023-12-10T18:55:50.791Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.659827936Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151812023-12-10T18:55:50.791Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.659880771Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151822023-12-10T18:55:50.791Z{"msg":"[1] 9119e673-14cb-443c-80ab-6739285d75eb (a511b1ae-51ec-4f52-a8a9-80702ead4569) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.660554475Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151832023-12-10T18:55:50.792Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.66060268Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151842023-12-10T18:55:50.792Z{"msg":"[1] client is_active_req TRUE, promote! session a511b1ae-51ec-4f52-a8a9-80702ead4569","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.660642968Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151852023-12-10T18:55:50.792Z{"msg":"[0] 9119e673-14cb-443c-80ab-6739285d75eb (a511b1ae-51ec-4f52-a8a9-80702ead4569) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.660845685Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151862023-12-10T18:55:50.792Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.66092526Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151872023-12-10T18:55:50.792Z{"msg":"[0] client is_active_req TRUE, promote! session a511b1ae-51ec-4f52-a8a9-80702ead4569","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.660972975Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151882023-12-10T18:55:50.792Z{"msg":"[2] 9119e673-14cb-443c-80ab-6739285d75eb (a511b1ae-51ec-4f52-a8a9-80702ead4569) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.661058056Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151892023-12-10T18:55:50.793Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.6611301Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151902023-12-10T18:55:50.793Z{"msg":"[2] client is_active_req TRUE, promote! session a511b1ae-51ec-4f52-a8a9-80702ead4569","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.661175168Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151912023-12-10T18:55:50.793Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.661365415Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151922023-12-10T18:55:50.793Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.661447609Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151932023-12-10T18:55:50.793Z{"msg":"9119e673-14cb-443c-80ab-6739285d75eb WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.661523915Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151942023-12-10T18:55:50.793Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.661593661Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151952023-12-10T18:55:50.794Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.661715857Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151962023-12-10T18:55:50.794Z{"msg":"9119e673-14cb-443c-80ab-6739285d75eb WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.661753979Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151972023-12-10T18:55:50.794Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.661973738Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151982023-12-10T18:55:50.794Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.662056951Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
151992023-12-10T18:55:50.794Z{"msg":"9119e673-14cb-443c-80ab-6739285d75eb WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.662135731Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152002023-12-10T18:55:50.794Z{"msg":"[0] 9119e673-14cb-443c-80ab-6739285d75eb (a511b1ae-51ec-4f52-a8a9-80702ead4569) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.662269022Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152012023-12-10T18:55:50.794Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.662349927Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152022023-12-10T18:55:50.794Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:50.662393396Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152032023-12-10T18:55:50.795Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.662491579Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152042023-12-10T18:55:50.795Z{"msg":"[1] 9119e673-14cb-443c-80ab-6739285d75eb (a511b1ae-51ec-4f52-a8a9-80702ead4569) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.6625381Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152052023-12-10T18:55:50.795Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.662611837Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152062023-12-10T18:55:50.795Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:50.662654136Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152072023-12-10T18:55:50.795Z{"msg":"9119e673-14cb-443c-80ab-6739285d75eb WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.662686412Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152082023-12-10T18:55:50.795Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.6627541Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152092023-12-10T18:55:50.795Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.662794143Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152102023-12-10T18:55:50.796Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.662850812Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152112023-12-10T18:55:50.796Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.662875357Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152122023-12-10T18:55:50.796Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.662916114Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152132023-12-10T18:55:50.796Z{"msg":"9119e673-14cb-443c-80ab-6739285d75eb WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.662939261Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152142023-12-10T18:55:50.796Z{"msg":"[2] 9119e673-14cb-443c-80ab-6739285d75eb (a511b1ae-51ec-4f52-a8a9-80702ead4569) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.662983302Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152152023-12-10T18:55:50.796Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.663005745Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152162023-12-10T18:55:50.796Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:50.663045652Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152172023-12-10T18:55:50.796Z{"msg":"[0]R flush_numbers[0..12]: [205, 214, 190, 213, 215, 215, 215, 212, 208, 202, 197, 214]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.663068888Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152182023-12-10T18:55:50.796Z{"msg":"[0]R generation[0..12]: [117, 122, 108, 121, 123, 123, 123, 120, 119, 114, 112, 122]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.663118155Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152192023-12-10T18:55:50.797Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.663142023Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152202023-12-10T18:55:50.797Z{"msg":"[1]R flush_numbers[0..12]: [205, 214, 190, 213, 215, 215, 215, 212, 208, 202, 197, 214]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.663163183Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152212023-12-10T18:55:50.797Z{"msg":"[1]R generation[0..12]: [117, 122, 108, 121, 123, 123, 123, 120, 119, 114, 112, 122]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.663225098Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152222023-12-10T18:55:50.797Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.66325129Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152232023-12-10T18:55:50.797Z{"msg":"[2]R flush_numbers[0..12]: [205, 214, 190, 213, 215, 215, 215, 212, 208, 202, 197, 214]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.663273048Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152242023-12-10T18:55:50.837Z{"msg":"[2]R generation[0..12]: [117, 122, 108, 121, 123, 123, 123, 120, 119, 114, 112, 122]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.663299878Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152252023-12-10T18:55:50.837Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:124
152262023-12-10T18:55:50.837ZWait for a query_work_queue command to finish before sending IO
152272023-12-10T18:55:50.838Z,"time":"2023-12-10T18:55:50.663380586Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152282023-12-10T18:55:50.838Z{"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.663425525Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152292023-12-10T18:55:50.838Z{"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.663467546Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152302023-12-10T18:55:50.838Z{"msg":"Next flush: 216","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.663504175Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152312023-12-10T18:55:50.838Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.663572544Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152322023-12-10T18:55:50.838Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.663610708Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152332023-12-10T18:55:50.839Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.66370079Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152342023-12-10T18:55:50.839Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.663743811Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152352023-12-10T18:55:50.839Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.663782509Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152362023-12-10T18:55:50.839Z{"msg":"9119e673-14cb-443c-80ab-6739285d75eb is now active with session: a511b1ae-51ec-4f52-a8a9-80702ead4569","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.663817856Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152372023-12-10T18:55:50.839Z{"msg":"9119e673-14cb-443c-80ab-6739285d75eb Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.66384131Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152382023-12-10T18:55:50.841Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.663868279Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152392023-12-10T18:55:50.841Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.663913158Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152402023-12-10T18:55:50.841Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.663936591Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152412023-12-10T18:55:50.841Z{"msg":"9119e673-14cb-443c-80ab-6739285d75eb Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.663957137Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152422023-12-10T18:55:50.841Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.664004219Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152432023-12-10T18:55:50.842Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.664025999Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152442023-12-10T18:55:50.842Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.664045734Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152452023-12-10T18:55:50.842Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.664105119Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152462023-12-10T18:55:50.842Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.664128843Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152472023-12-10T18:55:50.842Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.664154118Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152482023-12-10T18:55:50.842Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.664175055Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152492023-12-10T18:55:50.844Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.664194534Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152502023-12-10T18:55:50.844Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.66423558Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152512023-12-10T18:55:50.844Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:50.664257294Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1658,"session_id":"a511b1ae-51ec-4f52-a8a9-80702ead4569"}
152522023-12-10T18:55:50.844ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
152532023-12-10T18:55:50.844ZLoading write count information from file "/tmp/test_repair_verify.data"
152542023-12-10T18:55:50.844ZRun Repair workload
152552023-12-10T18:55:50.844Z01/30 Flush
152562023-12-10T18:55:50.845Z02/30 Write block 123 len 28672 data: 15 15 14 14 13 14 12
152572023-12-10T18:55:50.845Z03/30 Read block 476 len 8192
152582023-12-10T18:55:50.845Z04/30 Write block 30 len 36864 data: 8 9 12 8 9 9 12 10 9
152592023-12-10T18:55:50.845Z05/30 Read block 151 len 32768
152602023-12-10T18:55:50.845Z06/30 Flush
152612023-12-10T18:55:50.845Z07/30 Write block 566 len 24576 data: 12 12 10 11 12 11
152622023-12-10T18:55:50.845Z08/30 Read block 128 len 8192
152632023-12-10T18:55:50.845Z09/30 Read block 583 len 32768
152642023-12-10T18:55:50.845Z10/30 Write block 233 len 8192 data: 10 13
152652023-12-10T18:55:50.845Z11/30 Write block 279 len 24576 data: 10 8 6 9 10 11
152662023-12-10T18:55:50.845Z12/30 Read block 469 len 40960
152672023-12-10T18:55:50.846Z13/30 Write block 500 len 40960 data: 11 13 12 13 9 8 10 9 9 8
152682023-12-10T18:55:50.846Z14/30 Write block 400 len 16384 data: 9 10 10 11
152692023-12-10T18:55:50.846Z15/30 Write block 6 len 20480 data: 4 6 7 7 9
152702023-12-10T18:55:50.846Z16/30 Flush
152712023-12-10T18:55:50.846Z17/30 Write block 230 len 28672 data: 17 16 14 11 14 12 12
152722023-12-10T18:55:50.847Z18/30 Write block 180 len 20480 data: 4 6 8 9 7
152732023-12-10T18:55:50.847Z19/30 Read block 536 len 16384
152742023-12-10T18:55:50.847Z20/30 Read block 335 len 24576
152752023-12-10T18:55:50.847Z21/30 Read block 177 len 28672
152762023-12-10T18:55:50.847Z22/30 Read block 489 len 4096
152772023-12-10T18:55:50.847Z23/30 Read block 49 len 32768
152782023-12-10T18:55:50.847Z24/30 Read block 579 len 28672
152792023-12-10T18:55:50.847Z25/30 Read block 424 len 20480
152802023-12-10T18:55:50.847Z26/30 Read block 211 len 24576
152812023-12-10T18:55:50.847Z27/30 Write block 192 len 4096 data: 10
152822023-12-10T18:55:50.847Z28/30 Write block 393 len 24576 data: 7 6 6 6 5 7
152832023-12-10T18:55:50.848Z29/30 Read block 499 len 16384
152842023-12-10T18:55:50.848Z30/30 Read block 565 len 32768
152852023-12-10T18:55:50.848Z----------------------------------------------------------------
152862023-12-10T18:55:50.848Z Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:29
152872023-12-10T18:55:50.848ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
152882023-12-10T18:55:50.848Z 2 Acked 1001 Write 7 Done Done Done false
152892023-12-10T18:55:50.848Z 3 Acked 1002 Read 2 Done Done Done false
152902023-12-10T18:55:50.849Z 4 Acked 1003 Write 9 Done Done Done false
152912023-12-10T18:55:50.849Z 5 Acked 1004 Read 8 Done Done Sent false
152922023-12-10T18:55:50.849Z 6 Acked 1005 Flush 0 Done Done Sent false
152932023-12-10T18:55:50.849Z 7 Acked 1006 Write 6 Done Done Sent false
152942023-12-10T18:55:50.849Z 8 Acked 1007 Read 2 Done Done Sent false
152952023-12-10T18:55:50.849Z 9 Acked 1008 Read 8 Done Done Sent false
152962023-12-10T18:55:50.849Z 10 Acked 1009 Write 2 Done Done Sent false
152972023-12-10T18:55:50.849Z 11 Acked 1010 Write 6 Done Done Sent false
152982023-12-10T18:55:50.850Z 12 Acked 1011 Read 10 Done Done Sent false
152992023-12-10T18:55:50.850Z 13 Acked 1012 Write 10 Done Done Sent false
153002023-12-10T18:55:50.850Z 14 Acked 1013 Write 4 Done Done Sent false
153012023-12-10T18:55:50.850Z 15 Acked 1014 Write 5 Done Done Sent false
153022023-12-10T18:55:50.850Z 16 Acked 1015 Flush 0 Done Done Sent false
153032023-12-10T18:55:50.850Z 17 Acked 1016 Write 7 Done Done Sent false
153042023-12-10T18:55:50.851Z 18 Acked 1017 Write 5 Done Done Sent false
153052023-12-10T18:55:50.851Z 19 Acked 1018 Read 4 Done Done Sent false
153062023-12-10T18:55:50.851Z 20 Acked 1019 Read 6 Done Done Sent false
153072023-12-10T18:55:50.851Z 21 Acked 1020 Read 7 Done Done Sent false
153082023-12-10T18:55:50.851Z 22 Acked 1021 Read 1 Done Done Sent false
153092023-12-10T18:55:50.851Z 23 Acked 1022 Read 8 Done Done Sent false
153102023-12-10T18:55:50.851Z 24 Acked 1023 Read 7 Done Done Sent false
153112023-12-10T18:55:50.851Z 25 Acked 1024 Read 5 Done Done Sent false
153122023-12-10T18:55:50.852Z 26 Acked 1025 Read 6 Done Done Sent false
153132023-12-10T18:55:50.852Z 27 Acked 1026 Write 1 Done Done Sent false
153142023-12-10T18:55:50.852Z 28 Acked 1027 Write 6 Done Done Sent false
153152023-12-10T18:55:50.852Z 29 Acked 1028 Read 4 Done Done Sent false
153162023-12-10T18:55:50.852Z 30 Acked 1029 Read 8 Done Sent Sent false
153172023-12-10T18:55:50.852Z STATES DS:0 DS:1 DS:2 TOTAL
153182023-12-10T18:55:50.852Z New 0 0 0 0
153192023-12-10T18:55:50.852Z Sent 0 1 26 27
153202023-12-10T18:55:50.853Z Done 29 28 3 60
153212023-12-10T18:55:50.853Z Skipped 0 0 0 0
153222023-12-10T18:55:50.853Z Error 0 0 0 0
153232023-12-10T18:55:50.853ZLast Flush: 1015 1015 1000
153242023-12-10T18:55:50.873ZDownstairs last five completed: 1000
153252023-12-10T18:55:50.874ZUpstairs last five completed: 30 29 28 27 26 25
153262023-12-10T18:55:50.874ZWrote out file "/tmp/test_repair_verify.data"
153272023-12-10T18:55:50.874Z
153282023-12-10T18:55:50.874ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
153292023-12-10T18:55:50.874ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
153302023-12-10T18:55:50.875Z 0 000-019 124 124 117 218 218 205 F F F <---
153312023-12-10T18:55:50.875Z 1 020-039 124 124 122 217 217 214 F F T <---
153322023-12-10T18:55:50.875Z 2 040-059 108 108 108 190 190 190 F F F
153332023-12-10T18:55:50.876Z 3 060-079 121 121 121 213 213 213 F F F
153342023-12-10T18:55:50.876Z 4 080-099 123 123 123 215 215 215 F F F
153352023-12-10T18:55:50.876Z 5 100-119 123 123 123 215 215 215 F F F
153362023-12-10T18:55:50.876Z 6 120-139 124 124 123 217 217 215 F F T <---
153372023-12-10T18:55:50.877Z 7 140-159 120 120 120 212 212 212 F F F
153382023-12-10T18:55:50.877Z 8 160-179 119 119 119 208 208 208 F F F
153392023-12-10T18:55:50.877Z 9 180-199 114 114 114 202 202 202 T T F <---
153402023-12-10T18:55:50.877Z 10 200-219 112 112 112 197 197 197 F F F
153412023-12-10T18:55:50.877Z 11 220-239 124 124 122 218 218 214 T T F <---
153422023-12-10T18:55:50.878Z 12 240-259 114 114 114 202 202 202 F F F
153432023-12-10T18:55:50.878Z 13 260-279 124 124 122 218 218 214 F F F <---
153442023-12-10T18:55:50.878Z 14 280-299 124 124 122 218 218 214 F F F <---
153452023-12-10T18:55:50.878Z 15 300-319 122 122 122 214 214 214 F F F
153462023-12-10T18:55:50.878Z 16 320-339 114 114 114 202 202 202 F F F
153472023-12-10T18:55:50.878Z 17 340-359 122 122 122 214 214 214 F F F
153482023-12-10T18:55:50.879Z 18 360-379 122 122 122 214 214 214 F F F
153492023-12-10T18:55:50.879Z 19 380-399 117 117 117 205 205 205 T T F <---
153502023-12-10T18:55:50.879Z 20 400-419 124 124 123 218 218 215 F F F <---
153512023-12-10T18:55:50.879Z 21 420-439 108 108 108 189 189 189 F F F
153522023-12-10T18:55:50.879Z 22 440-459 122 122 122 214 214 214 F F F
153532023-12-10T18:55:50.879Z 23 460-479 123 123 123 215 215 215 F F F
153542023-12-10T18:55:50.879Z 24 480-499 123 123 123 215 215 215 F F F
153552023-12-10T18:55:50.879Z 25 500-519 124 124 116 218 218 203 F F F <---
153562023-12-10T18:55:50.879Z 26 520-539 122 122 122 214 214 214 F F F
153572023-12-10T18:55:50.879Z 27 540-559 116 116 116 204 204 204 F F F
153582023-12-10T18:55:50.880Z 28 560-579 124 124 120 218 218 210 F F F <---
153592023-12-10T18:55:50.880Z 29 580-599 119 119 119 208 208 208 F F F
153602023-12-10T18:55:50.880ZMax gen: 124, Max flush: 218
153612023-12-10T18:55:50.880ZError: Difference in extent metadata found!
153622023-12-10T18:55:50.880ZA Difference in extent metadata is expected here
153632023-12-10T18:55:50.880ZOn loop 62
153642023-12-10T18:55:50.880Z
153652023-12-10T18:55:50.880ZVerifying data now
153662023-12-10T18:55:55.861ZNo change after verify
153672023-12-10T18:55:55.861ZLoop: 62 Downstairs dump after verify (and repair):
153682023-12-10T18:55:55.893ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
153692023-12-10T18:55:55.893Z 0 000-019 124 124 124 218 218 218 F F F
153702023-12-10T18:55:56.326Z 1 020-039 124 124 124 217 217 217 F F F
153712023-12-10T18:55:56.326Z 2 040-059 108 108 108 190 190 190 F F F
153722023-12-10T18:55:56.326Z 3 060-079 121 121 121 213 213 213 F F F
153732023-12-10T18:55:56.327Z 4 080-099 123 123 123 215 215 215 F F F
153742023-12-10T18:55:56.327Z 5 100-119 123 123 123 215 215 215 F F F
153752023-12-10T18:55:56.327Z 6 120-139 124 124 124 217 217 217 F F F
153762023-12-10T18:55:56.327Z 7 140-159 120 120 120 212 212 212 F F F
153772023-12-10T18:55:56.328Z 8 160-179 119 119 119 208 208 208 F F F
153782023-12-10T18:55:56.328Z 9 180-199 125 125 125 219 219 219 F F F
153792023-12-10T18:55:56.328Z 10 200-219 112 112 112 197 197 197 F F F
153802023-12-10T18:55:56.329Z 11 220-239 125 125 125 219 219 219 F F F
153812023-12-10T18:55:56.329Z 12 240-259 114 114 114 202 202 202 F F F
153822023-12-10T18:55:56.329Z 13 260-279 124 124 124 218 218 218 F F F
153832023-12-10T18:55:56.329Z 14 280-299 124 124 124 218 218 218 F F F
153842023-12-10T18:55:56.330Z 15 300-319 122 122 122 214 214 214 F F F
153852023-12-10T18:55:56.330Z 16 320-339 114 114 114 202 202 202 F F F
153862023-12-10T18:55:56.330Z 17 340-359 122 122 122 214 214 214 F F F
153872023-12-10T18:55:56.331Z 18 360-379 122 122 122 214 214 214 F F F
153882023-12-10T18:55:56.331Z 19 380-399 125 125 125 219 219 219 F F F
153892023-12-10T18:55:56.331Z 20 400-419 124 124 124 218 218 218 F F F
153902023-12-10T18:55:56.332Z 21 420-439 108 108 108 189 189 189 F F F
153912023-12-10T18:55:56.332Z 22 440-459 122 122 122 214 214 214 F F F
153922023-12-10T18:55:56.332Z 23 460-479 123 123 123 215 215 215 F F F
153932023-12-10T18:55:56.333Z 24 480-499 123 123 123 215 215 215 F F F
153942023-12-10T18:55:56.333Z 25 500-519 124 124 124 218 218 218 F F F
153952023-12-10T18:55:56.333Z 26 520-539 122 122 122 214 214 214 F F F
153962023-12-10T18:55:56.334Z 27 540-559 116 116 116 204 204 204 F F F
153972023-12-10T18:55:56.334Z 28 560-579 124 124 124 218 218 218 F F F
153982023-12-10T18:55:56.334Z 29 580-599 119 119 119 208 208 208 F F F
153992023-12-10T18:55:56.334ZMax gen: 125, Max flush: 219
154002023-12-10T18:55:56.334Z
154012023-12-10T18:55:56.334ZBegin loop 63
154022023-12-10T18:55:56.335ZDownstairs to restart: 1
154032023-12-10T18:55:56.384Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.919179861Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666}
154042023-12-10T18:55:56.384Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.919630401Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666}
154052023-12-10T18:55:56.384Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.91969656Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666}
154062023-12-10T18:55:56.385Z{"msg":"Crucible d0ba6b2b-8c86-4a8b-8e3f-2fd9632167fb has session id: 17affad0-f728-4f1f-81b3-a5953a9867a9","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.91973726Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154072023-12-10T18:55:56.385ZCrucible runtime is spawned
154082023-12-10T18:55:56.385Z{"msg":"Upstairs opts: Upstairs UUID: d0ba6b2b-8c86-4a8b-8e3f-2fd9632167fb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30The guest has requested activation with gen:126
154092023-12-10T18:55:56.385Z,"time":"2023-12-10T18:55:55.919937482Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154102023-12-10T18:55:56.386Z{"msg":"Crucible stats registered with UUID: d0ba6b2b-8c86-4a8b-8e3f-2fd9632167fb","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.920084381Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154112023-12-10T18:55:56.387Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.920202862Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9","task":"up_listen"}
154122023-12-10T18:55:56.387Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.920477492Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154132023-12-10T18:55:56.387Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.920561463Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154142023-12-10T18:55:56.387Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.920602438Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"looper":"0","session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154152023-12-10T18:55:56.388Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.920671435Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"looper":"1","session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154162023-12-10T18:55:56.388Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.920742691Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"looper":"2","session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154172023-12-10T18:55:56.388Z{"msg":"d0ba6b2b-8c86-4a8b-8e3f-2fd9632167fb active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.920812552Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154182023-12-10T18:55:56.388Z{"msg":"Set desired generation to :126","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.920878672Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154192023-12-10T18:55:56.388Z{"msg":"[0] d0ba6b2b-8c86-4a8b-8e3f-2fd9632167fb looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.920916093Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"looper":"0","session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154202023-12-10T18:55:56.389Z{"msg":"[2] d0ba6b2b-8c86-4a8b-8e3f-2fd9632167fb looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.920997625Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"looper":"2","session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154212023-12-10T18:55:56.391Z{"msg":"[1] d0ba6b2b-8c86-4a8b-8e3f-2fd9632167fb looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.921040589Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"looper":"1","session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154222023-12-10T18:55:56.391Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.921108184Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154232023-12-10T18:55:56.391Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.921180637Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154242023-12-10T18:55:56.391Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.921225954Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154252023-12-10T18:55:56.392Z{"msg":"[0] d0ba6b2b-8c86-4a8b-8e3f-2fd9632167fb (17affad0-f728-4f1f-81b3-a5953a9867a9) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.921269239Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154262023-12-10T18:55:56.392Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.92134151Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154272023-12-10T18:55:56.392Z{"msg":"[0] client is_active_req TRUE, promote! session 17affad0-f728-4f1f-81b3-a5953a9867a9","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.921383437Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154282023-12-10T18:55:56.392Z{"msg":"[2] d0ba6b2b-8c86-4a8b-8e3f-2fd9632167fb (17affad0-f728-4f1f-81b3-a5953a9867a9) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.921465486Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154292023-12-10T18:55:56.392Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.921504977Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154302023-12-10T18:55:56.392Z{"msg":"[2] client is_active_req TRUE, promote! session 17affad0-f728-4f1f-81b3-a5953a9867a9","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.921541403Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154312023-12-10T18:55:56.394Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.921930728Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154322023-12-10T18:55:56.395Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.922073411Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154332023-12-10T18:55:56.395Z{"msg":"d0ba6b2b-8c86-4a8b-8e3f-2fd9632167fb WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.922160966Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154342023-12-10T18:55:56.395Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.922293675Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154352023-12-10T18:55:56.395Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.922409638Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154362023-12-10T18:55:56.395Z{"msg":"d0ba6b2b-8c86-4a8b-8e3f-2fd9632167fb WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.922448752Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154372023-12-10T18:55:56.395Z{"msg":"[0] d0ba6b2b-8c86-4a8b-8e3f-2fd9632167fb (17affad0-f728-4f1f-81b3-a5953a9867a9) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.9227913Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154382023-12-10T18:55:56.396Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.922852597Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154392023-12-10T18:55:56.396Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:55.922877342Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154402023-12-10T18:55:56.396Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.922902298Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154412023-12-10T18:55:56.396Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.92297098Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154422023-12-10T18:55:56.396Z{"msg":"d0ba6b2b-8c86-4a8b-8e3f-2fd9632167fb WaitQuorum New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.923013769Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154432023-12-10T18:55:56.397Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.92307194Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154442023-12-10T18:55:56.397Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.923097373Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154452023-12-10T18:55:56.397Z{"msg":"[2] d0ba6b2b-8c86-4a8b-8e3f-2fd9632167fb (17affad0-f728-4f1f-81b3-a5953a9867a9) WaitQuorum New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.923207163Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154462023-12-10T18:55:56.397Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.923231878Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154472023-12-10T18:55:56.397Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:55.923295469Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154482023-12-10T18:55:56.398Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.923320965Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154492023-12-10T18:55:56.398Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.923377449Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154502023-12-10T18:55:56.398Z{"msg":"d0ba6b2b-8c86-4a8b-8e3f-2fd9632167fb WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.923403607Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154512023-12-10T18:55:56.398Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.923432368Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154522023-12-10T18:55:56.398Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.923492753Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154532023-12-10T18:55:56.398Z{"msg":"[1] d0ba6b2b-8c86-4a8b-8e3f-2fd9632167fb (17affad0-f728-4f1f-81b3-a5953a9867a9) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.923519496Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154542023-12-10T18:55:56.399Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.923578018Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154552023-12-10T18:55:56.399Z{"msg":"[1] client is_active_req TRUE, promote! session 17affad0-f728-4f1f-81b3-a5953a9867a9","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.923608926Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154562023-12-10T18:55:56.399Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.924067234Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154572023-12-10T18:55:56.399Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.924126492Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154582023-12-10T18:55:56.399Z{"msg":"d0ba6b2b-8c86-4a8b-8e3f-2fd9632167fb WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.924151539Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154592023-12-10T18:55:56.399Z{"msg":"[1] d0ba6b2b-8c86-4a8b-8e3f-2fd9632167fb (17affad0-f728-4f1f-81b3-a5953a9867a9) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.924662695Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154602023-12-10T18:55:56.400Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.924731589Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154612023-12-10T18:55:56.401Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:55:55.924757401Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154622023-12-10T18:55:56.401Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.924777664Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154632023-12-10T18:55:56.401Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.924800156Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154642023-12-10T18:55:56.401Z{"msg":"d0ba6b2b-8c86-4a8b-8e3f-2fd9632167fb WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.92482144Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154652023-12-10T18:55:56.401Z{"msg":"[0]R flush_numbers[0..12]: [218, 217, 190, 213, 215, 215, 217, 212, 208, 219, 197, 219]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.924843399Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154662023-12-10T18:55:56.402Z{"msg":"[0]R generation[0..12]: [124, 124, 108, 121, 123, 123, 124, 120, 119, 125, 112, 125]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.924885601Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154672023-12-10T18:55:56.402Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.924910104Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154682023-12-10T18:55:56.402Z{"msg":"[1]R flush_numbers[0..12]: [218, 217, 190, 213, 215, 215, 217, 212, 208, 219, 197, 219]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.924931649Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154692023-12-10T18:55:56.402Z{"msg":"[1]R generation[0..12]: [124, 124, 108, 121, 123, 123, 124, 120, 119, 125, 112, 125]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.924975264Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154702023-12-10T18:55:56.402Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.924999228Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154712023-12-10T18:55:56.402Z{"msg":"[2]R flush_numbers[0..12]: [218, 217, 190, 213, 215, 215, 217, 212, 208, 219, 197, 219]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.925020478Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154722023-12-10T18:55:56.402Z{"msg":"[2]R generation[0..12]: [124, 124, 108, 121, 123, 123, 124, 120, 119, 125, 112, 125]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.925041389Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154732023-12-10T18:55:56.402Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.925063125Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154742023-12-10T18:55:56.402Z{"msg":"Max found gen is 126","v":0,"name":"The guest has finished waiting for activation with:126
154752023-12-10T18:55:56.403Zcrucible","level":30Wait for a query_work_queue command to finish before sending IO
154762023-12-10T18:55:56.403Z,"time":"2023-12-10T18:55:55.925118372Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154772023-12-10T18:55:56.403Z{"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.925142761Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154782023-12-10T18:55:56.404Z{"msg":"Next flush: 220","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.925168278Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154792023-12-10T18:55:56.404Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.925236324Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154802023-12-10T18:55:56.404Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.925277179Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154812023-12-10T18:55:56.404Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.925345997Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154822023-12-10T18:55:56.404Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.925423523Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154832023-12-10T18:55:56.406Z{"msg":"d0ba6b2b-8c86-4a8b-8e3f-2fd9632167fb is now active with session: 17affad0-f728-4f1f-81b3-a5953a9867a9","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.925498519Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154842023-12-10T18:55:56.406Z{"msg":"d0ba6b2b-8c86-4a8b-8e3f-2fd9632167fb Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.925568101Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154852023-12-10T18:55:56.406Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.925639879Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154862023-12-10T18:55:56.406Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.925705407Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154872023-12-10T18:55:56.407Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.925778205Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154882023-12-10T18:55:56.407Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.925818993Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154892023-12-10T18:55:56.407Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.925883661Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154902023-12-10T18:55:56.407Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.925913248Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154912023-12-10T18:55:56.407Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.925975458Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154922023-12-10T18:55:56.408Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.926066329Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154932023-12-10T18:55:56.409Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.926100339Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154942023-12-10T18:55:56.409ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
154952023-12-10T18:55:56.409Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.926185554Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154962023-12-10T18:55:56.410Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:55:55.926234693Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1666,"session_id":"17affad0-f728-4f1f-81b3-a5953a9867a9"}
154972023-12-10T18:55:56.410ZLoading write count information from file "/tmp/test_repair_verify.data"
154982023-12-10T18:55:56.410ZRun Repair workload
154992023-12-10T18:55:56.411Z01/30 Write block 268 len 40960 data: 14 13 11 10 10 9 10 13 13 14
155002023-12-10T18:55:56.411Z02/30 Write block 60 len 32768 data: 5 4 6 7 8 10 11 10
155012023-12-10T18:55:56.411Z03/30 Write block 516 len 20480 data: 8 8 7 8 7
155022023-12-10T18:55:56.411Z04/30 Write block 556 len 40960 data: 6 4 6 8 8 10 12 10 11 11
155032023-12-10T18:55:56.427Z05/30 Write block 31 len 8192 data: 10 13
155042023-12-10T18:55:56.428Z06/30 Write block 100 len 32768 data: 9 9 11 10 7 7 8 8
155052023-12-10T18:55:56.428Z07/30 Write block 254 len 36864 data: 11 9 9 6 6 3 6 7 10
155062023-12-10T18:55:56.428Z08/30 Write block 73 len 40960 data: 11 13 12 12 14 13 15 14 15 15
155072023-12-10T18:55:56.428Z09/30 Read block 352 len 16384
155082023-12-10T18:55:56.428Z10/30 Write block 540 len 28672 data: 13 14 14 14 13 9 9
155092023-12-10T18:55:56.428Z11/30 Flush
155102023-12-10T18:55:56.428Z12/30 Write block 73 len 4096 data: 12
155112023-12-10T18:55:56.429Z13/30 Write block 68 len 8192 data: 13 13
155122023-12-10T18:55:56.429Z14/30 Read block 524 len 40960
155132023-12-10T18:55:56.429Z15/30 Write block 82 len 32768 data: 16 15 15 13 15 12 13 11
155142023-12-10T18:55:56.429Z16/30 Read block 350 len 24576
155152023-12-10T18:55:56.429Z17/30 Read block 544 len 36864
155162023-12-10T18:55:56.429Z18/30 Read block 390 len 4096
155172023-12-10T18:55:56.430Z19/30 Read block 335 len 36864
155182023-12-10T18:55:56.430Z20/30 Write block 397 len 4096 data: 6
155192023-12-10T18:55:56.430Z21/30 Read block 156 len 24576
155202023-12-10T18:55:56.430Z22/30 Write block 41 len 36864 data: 7 7 6 6 9 8 8 8 8
155212023-12-10T18:55:56.430Z23/30 Write block 8 len 40960 data: 8 8 10 8 10 13 15 14 11 12
155222023-12-10T18:55:56.430Z24/30 Read block 79 len 16384
155232023-12-10T18:55:56.430Z25/30 Read block 34 len 28672
155242023-12-10T18:55:56.431Z26/30 Read block 160 len 4096
155252023-12-10T18:55:56.431Z27/30 Read block 67 len 36864
155262023-12-10T18:55:56.431Z28/30 Write block 302 len 24576 data: 10 13 15 10 11 8
155272023-12-10T18:55:56.431Z29/30 Read block 352 len 4096
155282023-12-10T18:55:56.431Z30/30 Read block 551 len 40960
155292023-12-10T18:55:56.431Z----------------------------------------------------------------
155302023-12-10T18:55:56.432Z Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30
155312023-12-10T18:55:56.432ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
155322023-12-10T18:55:56.432Z 1 Acked 1000 Write 10 Done Done Done false
155332023-12-10T18:55:56.432Z 2 Acked 1001 Write 8 Done Done Done false
155342023-12-10T18:55:56.432Z 3 Acked 1002 Write 5 Done Done Done false
155352023-12-10T18:55:56.432Z 4 Acked 1003 Write 10 Done Done Done false
155362023-12-10T18:55:56.432Z 5 Acked 1004 Write 2 Done Done Done false
155372023-12-10T18:55:56.433Z 6 Acked 1005 Write 8 Done Done Done false
155382023-12-10T18:55:56.433Z 7 Acked 1006 Write 9 Done Done Done false
155392023-12-10T18:55:56.433Z 8 Acked 1007 Write 10 Done Done Done false
155402023-12-10T18:55:56.434Z 9 Acked 1008 Read 4 Done Sent Done false
155412023-12-10T18:55:56.434Z 10 Acked 1009 Write 7 Done Sent Done false
155422023-12-10T18:55:56.434Z 11 Acked 1010 Flush 0 Done Sent Done false
155432023-12-10T18:55:56.435Z 12 Acked 1011 Write 1 Done Sent Done false
155442023-12-10T18:55:56.435Z 13 Acked 1012 Write 2 Done Sent Done false
155452023-12-10T18:55:56.435Z 14 Acked 1013 Read 10 Done Sent Done false
155462023-12-10T18:55:56.435Z 15 Acked 1014 Write 8 Done Sent Done false
155472023-12-10T18:55:56.435Z 16 Acked 1015 Read 6 Done Sent Done false
155482023-12-10T18:55:56.435Z 17 Acked 1016 Read 9 Done Sent Done false
155492023-12-10T18:55:56.435Z 18 Acked 1017 Read 1 Done Sent Done false
155502023-12-10T18:55:56.436Z 19 Acked 1018 Read 9 Done Sent Done false
155512023-12-10T18:55:56.436Z 20 Acked 1019 Write 1 Done Sent Done false
155522023-12-10T18:55:56.436Z 21 Acked 1020 Read 6 Done Sent Done false
155532023-12-10T18:55:56.436Z 22 Acked 1021 Write 9 Done Sent Done false
155542023-12-10T18:55:56.436Z 23 Acked 1022 Write 10 Done Sent Done false
155552023-12-10T18:55:56.436Z 24 Acked 1023 Read 4 Done Sent Done false
155562023-12-10T18:55:56.436Z 25 Acked 1024 Read 7 Done Sent Done false
155572023-12-10T18:55:56.436Z 26 Acked 1025 Read 1 Done Sent Done false
155582023-12-10T18:55:56.436Z 27 Acked 1026 Read 9 Done Sent Done false
155592023-12-10T18:55:56.436Z 28 Acked 1027 Write 6 Done Sent Done false
155602023-12-10T18:55:56.437Z 29 Acked 1028 Read 1 Done Sent Done false
155612023-12-10T18:55:56.437Z 30 Acked 1029 Read 10 Sent Sent Done false
155622023-12-10T18:55:56.437Z STATES DS:0 DS:1 DS:2 TOTAL
155632023-12-10T18:55:56.437Z New 0 0 0 0
155642023-12-10T18:55:56.437Z Sent 1 22 0 23
155652023-12-10T18:55:56.437Z Done 29 8 30 67
155662023-12-10T18:55:56.437Z Skipped 0 0 0 0
155672023-12-10T18:55:56.438Z Error 0 0 0 0
155682023-12-10T18:55:56.438ZLast Flush: 1010 0 1010
155692023-12-10T18:55:56.438ZDownstairs last five completed:
155702023-12-10T18:55:56.438ZUpstairs last five completed: 30 29 28 27 26 25
155712023-12-10T18:55:56.438ZWrote out file "/tmp/test_repair_verify.data"
155722023-12-10T18:55:56.438Z
155732023-12-10T18:55:56.438ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
155742023-12-10T18:55:56.439ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
155752023-12-10T18:55:56.439Z 0 000-019 124 124 124 218 218 218 T F T <---
155762023-12-10T18:55:56.439Z 1 020-039 126 124 126 220 217 220 F T F <---
155772023-12-10T18:55:56.439Z 2 040-059 108 108 108 190 190 190 T F T <---
155782023-12-10T18:55:56.439Z 3 060-079 126 121 126 220 213 220 T T T <---
155792023-12-10T18:55:56.439Z 4 080-099 126 123 126 220 215 220 T T T <---
155802023-12-10T18:55:56.439Z 5 100-119 126 123 126 220 215 220 F T F <---
155812023-12-10T18:55:56.440Z 6 120-139 124 124 124 217 217 217 F F F
155822023-12-10T18:55:56.440Z 7 140-159 120 120 120 212 212 212 F F F
155832023-12-10T18:55:56.440Z 8 160-179 119 119 119 208 208 208 F F F
155842023-12-10T18:55:56.440Z 9 180-199 125 125 125 219 219 219 F F F
155852023-12-10T18:55:56.440Z 10 200-219 112 112 112 197 197 197 F F F
155862023-12-10T18:55:56.440Z 11 220-239 125 125 125 219 219 219 F F F
155872023-12-10T18:55:56.441Z 12 240-259 126 114 126 220 202 220 F T F <---
155882023-12-10T18:55:56.442Z 13 260-279 126 124 126 220 218 220 F T F <---
155892023-12-10T18:55:56.442Z 14 280-299 124 124 124 218 218 218 F F F
155902023-12-10T18:55:56.443Z 15 300-319 122 122 122 214 214 214 T F T <---
155912023-12-10T18:55:56.443Z 16 320-339 114 114 114 202 202 202 F F F
155922023-12-10T18:55:56.443Z 17 340-359 122 122 122 214 214 214 F F F
155932023-12-10T18:55:56.443Z 18 360-379 122 122 122 214 214 214 F F F
155942023-12-10T18:55:56.443Z 19 380-399 125 125 125 219 219 219 T F T <---
155952023-12-10T18:55:56.443Z 20 400-419 124 124 124 218 218 218 F F F
155962023-12-10T18:55:56.444Z 21 420-439 108 108 108 189 189 189 F F F
155972023-12-10T18:55:56.444Z 22 440-459 122 122 122 214 214 214 F F F
155982023-12-10T18:55:56.444Z 23 460-479 123 123 123 215 215 215 F F F
155992023-12-10T18:55:56.444Z 24 480-499 123 123 123 215 215 215 F F F
156002023-12-10T18:55:56.444Z 25 500-519 126 124 126 220 218 220 F T F <---
156012023-12-10T18:55:56.444Z 26 520-539 126 122 126 220 214 220 F T F <---
156022023-12-10T18:55:56.444Z 27 540-559 126 116 126 220 204 220 F T F <---
156032023-12-10T18:55:56.459Z 28 560-579 126 124 126 220 218 220 F T F <---
156042023-12-10T18:55:56.459Z 29 580-599 119 119 119 208 208 208 F F F
156052023-12-10T18:55:56.459ZMax gen: 126, Max flush: 220
156062023-12-10T18:55:56.459ZError: Difference in extent metadata found!
156072023-12-10T18:55:56.459ZA Difference in extent metadata is expected here
156082023-12-10T18:55:56.459ZOn loop 63
156092023-12-10T18:55:56.460Z
156102023-12-10T18:55:56.460ZVerifying data now
156112023-12-10T18:56:02.430ZNo change after verify
156122023-12-10T18:56:02.431ZLoop: 63 Downstairs dump after verify (and repair):
156132023-12-10T18:56:02.492ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
156142023-12-10T18:56:02.492Z 0 000-019 127 127 127 221 221 221 F F F
156152023-12-10T18:56:02.493Z 1 020-039 126 126 126 220 220 220 F F F
156162023-12-10T18:56:02.493Z 2 040-059 127 127 127 221 221 221 F F F
156172023-12-10T18:56:02.493Z 3 060-079 127 127 127 221 221 221 F F F
156182023-12-10T18:56:02.493Z 4 080-099 127 127 127 221 221 221 F F F
156192023-12-10T18:56:02.493Z 5 100-119 126 126 126 220 220 220 F F F
156202023-12-10T18:56:02.493Z 6 120-139 124 124 124 217 217 217 F F F
156212023-12-10T18:56:02.493Z 7 140-159 120 120 120 212 212 212 F F F
156222023-12-10T18:56:02.494Z 8 160-179 119 119 119 208 208 208 F F F
156232023-12-10T18:56:02.494Z 9 180-199 125 125 125 219 219 219 F F F
156242023-12-10T18:56:02.494Z 10 200-219 112 112 112 197 197 197 F F F
156252023-12-10T18:56:02.494Z 11 220-239 125 125 125 219 219 219 F F F
156262023-12-10T18:56:02.494Z 12 240-259 126 126 126 220 220 220 F F F
156272023-12-10T18:56:02.495Z 13 260-279 126 126 126 220 220 220 F F F
156282023-12-10T18:56:02.495Z 14 280-299 124 124 124 218 218 218 F F F
156292023-12-10T18:56:02.495Z 15 300-319 127 127 127 221 221 221 F F F
156302023-12-10T18:56:02.495Z 16 320-339 114 114 114 202 202 202 F F F
156312023-12-10T18:56:02.495Z 17 340-359 122 122 122 214 214 214 F F F
156322023-12-10T18:56:02.495Z 18 360-379 122 122 122 214 214 214 F F F
156332023-12-10T18:56:02.496Z 19 380-399 127 127 127 221 221 221 F F F
156342023-12-10T18:56:02.496Z 20 400-419 124 124 124 218 218 218 F F F
156352023-12-10T18:56:02.496Z 21 420-439 108 108 108 189 189 189 F F F
156362023-12-10T18:56:02.496Z 22 440-459 122 122 122 214 214 214 F F F
156372023-12-10T18:56:02.497Z 23 460-479 123 123 123 215 215 215 F F F
156382023-12-10T18:56:02.497Z 24 480-499 123 123 123 215 215 215 F F F
156392023-12-10T18:56:02.497Z 25 500-519 126 126 126 220 220 220 F F F
156402023-12-10T18:56:02.497Z 26 520-539 126 126 126 220 220 220 F F F
156412023-12-10T18:56:02.498Z 27 540-559 126 126 126 220 220 220 F F F
156422023-12-10T18:56:02.498Z 28 560-579 126 126 126 220 220 220 F F F
156432023-12-10T18:56:02.498Z 29 580-599 119 119 119 208 208 208 F F F
156442023-12-10T18:56:02.498ZMax gen: 127, Max flush: 221
156452023-12-10T18:56:02.498Z
156462023-12-10T18:56:02.499ZBegin loop 64
156472023-12-10T18:56:02.499ZDownstairs to restart: 2
156482023-12-10T18:56:02.499Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.42166685Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674}
156492023-12-10T18:56:02.499Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.422093268Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674}
156502023-12-10T18:56:02.499Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.422122752Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674}
156512023-12-10T18:56:02.500Z{"msg":"Crucible b8203201-39d4-4db8-964d-ed19517273fe has session id: c82617cf-4ccd-4b47-a121-ed3daad6d1de","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.422184496Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156522023-12-10T18:56:02.500Z{"msg":"Upstairs opts: Upstairs UUID: b8203201-39d4-4db8-964d-ed19517273fe, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.422275269Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156532023-12-10T18:56:02.500Z{"msg":"Crucible runtime is spawned
156542023-12-10T18:56:02.501ZCrucible stats registered with UUID: b8203201-39d4-4db8-964d-ed19517273fe","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.422501595Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156552023-12-10T18:56:02.501Z{"msg":"The guest has requested activation with gen:128
156562023-12-10T18:56:02.501Z[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.422663838Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"looper":"0","session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156572023-12-10T18:56:02.501Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.42275107Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de","task":"up_listen"}
156582023-12-10T18:56:02.501Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.422821996Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156592023-12-10T18:56:02.502Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.422909961Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156602023-12-10T18:56:02.502Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.422955076Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"looper":"1","session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156612023-12-10T18:56:02.502Z{"msg":"[0] b8203201-39d4-4db8-964d-ed19517273fe looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.423027837Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"looper":"0","session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156622023-12-10T18:56:02.502Z{"msg":"[1] b8203201-39d4-4db8-964d-ed19517273fe looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.423100553Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"looper":"1","session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156632023-12-10T18:56:02.502Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.423141895Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"looper":"2","session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156642023-12-10T18:56:02.502Z{"msg":"[2] b8203201-39d4-4db8-964d-ed19517273fe looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.423327177Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"looper":"2","session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156652023-12-10T18:56:02.503Z{"msg":"b8203201-39d4-4db8-964d-ed19517273fe active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.423651337Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156662023-12-10T18:56:02.503Z{"msg":"Set desired generation to :128","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.42370451Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156672023-12-10T18:56:02.503Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.423812488Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156682023-12-10T18:56:02.503Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.424138913Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156692023-12-10T18:56:02.503Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.424317389Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156702023-12-10T18:56:02.503Z{"msg":"[1] b8203201-39d4-4db8-964d-ed19517273fe (c82617cf-4ccd-4b47-a121-ed3daad6d1de) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.424604671Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156712023-12-10T18:56:02.504Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.424661869Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156722023-12-10T18:56:02.504Z{"msg":"[1] client is_active_req TRUE, promote! session c82617cf-4ccd-4b47-a121-ed3daad6d1de","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.424687668Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156732023-12-10T18:56:02.504Z{"msg":"[0] b8203201-39d4-4db8-964d-ed19517273fe (c82617cf-4ccd-4b47-a121-ed3daad6d1de) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.424779131Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156742023-12-10T18:56:02.504Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.424842043Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156752023-12-10T18:56:02.504Z{"msg":"[0] client is_active_req TRUE, promote! session c82617cf-4ccd-4b47-a121-ed3daad6d1de","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.424868933Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156762023-12-10T18:56:02.505Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.425457334Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156772023-12-10T18:56:02.505Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.425556005Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156782023-12-10T18:56:02.506Z{"msg":"b8203201-39d4-4db8-964d-ed19517273fe WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.425601346Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156792023-12-10T18:56:02.506Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.425643509Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156802023-12-10T18:56:02.506Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.425711937Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156812023-12-10T18:56:02.506Z{"msg":"b8203201-39d4-4db8-964d-ed19517273fe WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.425755159Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156822023-12-10T18:56:02.506Z{"msg":"[1] b8203201-39d4-4db8-964d-ed19517273fe (c82617cf-4ccd-4b47-a121-ed3daad6d1de) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.426388989Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156832023-12-10T18:56:02.507Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.426454685Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156842023-12-10T18:56:02.507Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:02.426496738Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156852023-12-10T18:56:02.508Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.426557221Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156862023-12-10T18:56:02.508Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.426598795Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156872023-12-10T18:56:02.508Z{"msg":"b8203201-39d4-4db8-964d-ed19517273fe WaitActive WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.426657548Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156882023-12-10T18:56:02.508Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.426698179Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156892023-12-10T18:56:02.509Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.426786359Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156902023-12-10T18:56:02.509Z{"msg":"[2] b8203201-39d4-4db8-964d-ed19517273fe (c82617cf-4ccd-4b47-a121-ed3daad6d1de) WaitActive WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.426832336Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156912023-12-10T18:56:02.509Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.4268694Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156922023-12-10T18:56:02.509Z{"msg":"[2] client is_active_req TRUE, promote! session c82617cf-4ccd-4b47-a121-ed3daad6d1de","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.426927036Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156932023-12-10T18:56:02.509Z{"msg":"[0] b8203201-39d4-4db8-964d-ed19517273fe (c82617cf-4ccd-4b47-a121-ed3daad6d1de) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.426971593Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156942023-12-10T18:56:02.510Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.427027928Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156952023-12-10T18:56:02.511Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:02.427068082Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156962023-12-10T18:56:02.511Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.427124993Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156972023-12-10T18:56:02.511Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.427165527Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156982023-12-10T18:56:02.512Z{"msg":"b8203201-39d4-4db8-964d-ed19517273fe WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.427224303Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
156992023-12-10T18:56:02.512Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.427265023Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157002023-12-10T18:56:02.512Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.427321742Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157012023-12-10T18:56:02.512Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.427413532Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157022023-12-10T18:56:02.513Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.427480504Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157032023-12-10T18:56:02.513Z{"msg":"b8203201-39d4-4db8-964d-ed19517273fe WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.427507278Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157042023-12-10T18:56:02.513Z{"msg":"[2] b8203201-39d4-4db8-964d-ed19517273fe (c82617cf-4ccd-4b47-a121-ed3daad6d1de) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.428101035Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157052023-12-10T18:56:02.514Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.428172899Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157062023-12-10T18:56:02.514Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:02.428198105Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157072023-12-10T18:56:02.515Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.428225918Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157082023-12-10T18:56:02.515Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.428286859Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157092023-12-10T18:56:02.515Z{"msg":"b8203201-39d4-4db8-964d-ed19517273fe WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.428314322Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157102023-12-10T18:56:02.515Z{"msg":"[0]R flush_numbers[0..12]: [221, 220, 221, 221, 221, 220, 217, 212, 208, 219, 197, 219]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.428358023Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157112023-12-10T18:56:02.516Z{"msg":"[0]R generation[0..12]: [127, 126, 127, 127, 127, 126, 124, 120, 119, 125, 112, 125]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.42838242Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157122023-12-10T18:56:02.516Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.428406477Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157132023-12-10T18:56:02.537Z{"msg":"[1]R flush_numbers[0..12]: [221, 220, 221, 221, 221, 220, 217, 212, 208, 219, 197, 219]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:128
157142023-12-10T18:56:02.537Z,"time":"2023-12-10T18:56:02.428467656Z","hostname":"Wait for a query_work_queue command to finish before sending IO
157152023-12-10T18:56:02.538Zip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157162023-12-10T18:56:02.538Z{"msg":"[1]R generation[0..12]: [127, 126, 127, 127, 127, 126, 124, 120, 119, 125, 112, 125]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.42853389Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157172023-12-10T18:56:02.538Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.428572835Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157182023-12-10T18:56:02.538Z{"msg":"[2]R flush_numbers[0..12]: [221, 220, 221, 221, 221, 220, 217, 212, 208, 219, 197, 219]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.428615842Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157192023-12-10T18:56:02.538Z{"msg":"[2]R generation[0..12]: [127, 126, 127, 127, 127, 126, 124, 120, 119, 125, 112, 125]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.428659605Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157202023-12-10T18:56:02.539Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.428745395Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157212023-12-10T18:56:02.539Z{"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.428787687Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157222023-12-10T18:56:02.539Z{"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.428850731Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157232023-12-10T18:56:02.539Z{"msg":"Next flush: 222","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.428876282Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157242023-12-10T18:56:02.539Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.428896188Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157252023-12-10T18:56:02.540Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.428945144Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157262023-12-10T18:56:02.540Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.428967784Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157272023-12-10T18:56:02.542Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.428988107Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157282023-12-10T18:56:02.542Z{"msg":"b8203201-39d4-4db8-964d-ed19517273fe is now active with session: c82617cf-4ccd-4b47-a121-ed3daad6d1de","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.429009944Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157292023-12-10T18:56:02.542Z{"msg":"b8203201-39d4-4db8-964d-ed19517273fe Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.429055618Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157302023-12-10T18:56:02.542Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.429078681Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157312023-12-10T18:56:02.542Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.429098549Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157322023-12-10T18:56:02.543Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.429138866Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157332023-12-10T18:56:02.543Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.429160969Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157342023-12-10T18:56:02.543Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.429180996Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157352023-12-10T18:56:02.543Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.429221348Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157362023-12-10T18:56:02.543Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.429243204Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157372023-12-10T18:56:02.543Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.42926306Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157382023-12-10T18:56:02.544Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.429305022Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157392023-12-10T18:56:02.544Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.429327402Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157402023-12-10T18:56:02.545Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:02.429346775Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1674,"session_id":"c82617cf-4ccd-4b47-a121-ed3daad6d1de"}
157412023-12-10T18:56:02.545ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
157422023-12-10T18:56:02.545ZLoading write count information from file "/tmp/test_repair_verify.data"
157432023-12-10T18:56:02.545ZRun Repair workload
157442023-12-10T18:56:02.546Z01/30 Write block 410 len 20480 data: 9 7 5 4 5
157452023-12-10T18:56:02.546Z02/30 Read block 440 len 32768
157462023-12-10T18:56:02.546Z03/30 Write block 408 len 16384 data: 13 10 10 8
157472023-12-10T18:56:02.546Z04/30 Write block 592 len 4096 data: 10
157482023-12-10T18:56:02.546Z05/30 Read block 33 len 36864
157492023-12-10T18:56:02.546Z06/30 Flush
157502023-12-10T18:56:02.546Z07/30 Write block 590 len 12288 data: 12 12 11
157512023-12-10T18:56:02.546Z08/30 Write block 505 len 20480 data: 9 11 10 10 9
157522023-12-10T18:56:02.547Z09/30 Read block 384 len 16384
157532023-12-10T18:56:02.547Z10/30 Write block 79 len 4096 data: 16
157542023-12-10T18:56:02.547Z11/30 Write block 323 len 32768 data: 11 11 9 10 11 10 9 10
157552023-12-10T18:56:02.547Z12/30 Read block 359 len 4096
157562023-12-10T18:56:02.547Z13/30 Read block 559 len 36864
157572023-12-10T18:56:02.548Z14/30 Write block 148 len 28672 data: 7 8 7 6 6 6 11
157582023-12-10T18:56:02.548Z15/30 Write block 95 len 8192 data: 12 9
157592023-12-10T18:56:02.548Z16/30 Read block 120 len 16384
157602023-12-10T18:56:02.548Z17/30 Flush
157612023-12-10T18:56:02.548Z18/30 Write block 536 len 28672 data: 8 9 9 9 14 15 15
157622023-12-10T18:56:02.548Z19/30 Read block 32 len 40960
157632023-12-10T18:56:02.549Z20/30 Read block 532 len 20480
157642023-12-10T18:56:02.549Z21/30 Read block 33 len 12288
157652023-12-10T18:56:02.549Z22/30 Read block 73 len 32768
157662023-12-10T18:56:02.549Z23/30 Write block 120 len 12288 data: 13 13 13
157672023-12-10T18:56:02.549Z24/30 Read block 199 len 36864
157682023-12-10T18:56:02.549Z25/30 Flush
157692023-12-10T18:56:02.550Z26/30 Write block 533 len 40960 data: 11 13 14 9 10 10 10 15 16 16
157702023-12-10T18:56:02.550Z27/30 Write block 417 len 4096 data: 9
157712023-12-10T18:56:02.550Z28/30 Write block 226 len 4096 data: 13
157722023-12-10T18:56:02.550Z29/30 Read block 466 len 24576
157732023-12-10T18:56:02.550Z30/30 Write block 485 len 16384 data: 8 10 10 11
157742023-12-10T18:56:02.550Z----------------------------------------------------------------
157752023-12-10T18:56:02.550Z Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:24
157762023-12-10T18:56:02.551ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
157772023-12-10T18:56:02.551Z 7 Acked 1006 Write 3 Done Done Done false
157782023-12-10T18:56:02.551Z 8 Acked 1007 Write 5 Done Done Done false
157792023-12-10T18:56:02.551Z 9 Acked 1008 Read 4 Done Done Sent false
157802023-12-10T18:56:02.552Z 10 Acked 1009 Write 1 Done Done Sent false
157812023-12-10T18:56:02.552Z 11 Acked 1010 Write 8 Done Done Sent false
157822023-12-10T18:56:02.552Z 12 Acked 1011 Read 1 Done Done Sent false
157832023-12-10T18:56:02.553Z 13 Acked 1012 Read 9 Done Done Sent false
157842023-12-10T18:56:02.553Z 14 Acked 1013 Write 7 Done Done Sent false
157852023-12-10T18:56:02.553Z 15 Acked 1014 Write 2 Done Done Sent false
157862023-12-10T18:56:02.554Z 16 Acked 1015 Read 4 Done Done Sent false
157872023-12-10T18:56:02.554Z 17 Acked 1016 Flush 0 Done Done Sent false
157882023-12-10T18:56:02.554Z 18 Acked 1017 Write 7 Done Done Sent false
157892023-12-10T18:56:02.554Z 19 Acked 1018 Read 10 Done Done Sent false
157902023-12-10T18:56:02.555Z 20 Acked 1019 Read 5 Done Done Sent false
157912023-12-10T18:56:02.555Z 21 Acked 1020 Read 3 Done Done Sent false
157922023-12-10T18:56:02.555Z 22 Acked 1021 Read 8 Done Done Sent false
157932023-12-10T18:56:02.555Z 23 Acked 1022 Write 3 Done Done Sent false
157942023-12-10T18:56:02.555Z 24 Acked 1023 Read 9 Done Done Sent false
157952023-12-10T18:56:02.555Z 25 Acked 1024 Flush 0 Done Done Sent false
157962023-12-10T18:56:02.556Z 26 Acked 1025 Write 10 Done Done Sent false
157972023-12-10T18:56:02.556Z 27 Acked 1026 Write 1 Done Done Sent false
157982023-12-10T18:56:02.556Z 28 Acked 1027 Write 1 Done Done Sent false
157992023-12-10T18:56:02.557Z 29 Acked 1028 Read 6 Done Done Sent false
158002023-12-10T18:56:02.557Z 30 Acked 1029 Write 4 Sent Sent Sent false
158012023-12-10T18:56:02.557Z STATES DS:0 DS:1 DS:2 TOTAL
158022023-12-10T18:56:02.557Z New 0 0 0 0
158032023-12-10T18:56:02.558Z Sent 1 1 22 24
158042023-12-10T18:56:02.558Z Done 23 23 2 48
158052023-12-10T18:56:02.558Z Skipped 0 0 0 0
158062023-12-10T18:56:02.558Z Error 0 0 0 0
158072023-12-10T18:56:02.558ZLast Flush: 1024 1024 1005
158082023-12-10T18:56:02.558ZDownstairs last five completed: 1005 1004 1003 1002 1001 1000
158092023-12-10T18:56:02.558ZUpstairs last five completed: 30 29 28 27 26 25
158102023-12-10T18:56:02.558ZWrote out file "/tmp/test_repair_verify.data"
158112023-12-10T18:56:02.558Z
158122023-12-10T18:56:02.559ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
158132023-12-10T18:56:02.580ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
158142023-12-10T18:56:02.580Z 0 000-019 127 127 127 221 221 221 F F F
158152023-12-10T18:56:02.580Z 1 020-039 126 126 126 220 220 220 F F F
158162023-12-10T18:56:02.581Z 2 040-059 127 127 127 221 221 221 F F F
158172023-12-10T18:56:02.581Z 3 060-079 128 128 127 223 223 221 F F F <---
158182023-12-10T18:56:02.581Z 4 080-099 128 128 127 223 223 221 F F F <---
158192023-12-10T18:56:02.581Z 5 100-119 126 126 126 220 220 220 F F F
158202023-12-10T18:56:02.582Z 6 120-139 128 128 124 224 224 217 F F F <---
158212023-12-10T18:56:02.582Z 7 140-159 128 128 120 223 223 212 F F F <---
158222023-12-10T18:56:02.582Z 8 160-179 119 119 119 208 208 208 F F F
158232023-12-10T18:56:02.583Z 9 180-199 125 125 125 219 219 219 F F F
158242023-12-10T18:56:02.583Z 10 200-219 112 112 112 197 197 197 F F F
158252023-12-10T18:56:02.583Z 11 220-239 125 125 125 219 219 219 T T F <---
158262023-12-10T18:56:02.583Z 12 240-259 126 126 126 220 220 220 F F F
158272023-12-10T18:56:02.583Z 13 260-279 126 126 126 220 220 220 F F F
158282023-12-10T18:56:02.584Z 14 280-299 124 124 124 218 218 218 F F F
158292023-12-10T18:56:02.584Z 15 300-319 127 127 127 221 221 221 F F F
158302023-12-10T18:56:02.584Z 16 320-339 128 128 114 223 223 202 F F F <---
158312023-12-10T18:56:02.584Z 17 340-359 122 122 122 214 214 214 F F F
158322023-12-10T18:56:02.585Z 18 360-379 122 122 122 214 214 214 F F F
158332023-12-10T18:56:02.585Z 19 380-399 127 127 127 221 221 221 F F F
158342023-12-10T18:56:02.585Z 20 400-419 128 128 128 222 222 222 T T F <---
158352023-12-10T18:56:02.585Z 21 420-439 108 108 108 189 189 189 F F F
158362023-12-10T18:56:02.586Z 22 440-459 122 122 122 214 214 214 F F F
158372023-12-10T18:56:02.586Z 23 460-479 123 123 123 215 215 215 F F F
158382023-12-10T18:56:02.586Z 24 480-499 123 123 123 215 215 215 T T F <---
158392023-12-10T18:56:02.586Z 25 500-519 128 128 126 223 223 220 F F T <---
158402023-12-10T18:56:02.586Z 26 520-539 128 128 126 224 224 220 T T F <---
158412023-12-10T18:56:02.587Z 27 540-559 128 128 126 224 224 220 T T F <---
158422023-12-10T18:56:02.587Z 28 560-579 126 126 126 220 220 220 F F F
158432023-12-10T18:56:02.587Z 29 580-599 128 128 128 223 223 222 F F T <---
158442023-12-10T18:56:02.587ZMax gen: 128, Max flush: 224
158452023-12-10T18:56:02.588ZError: Difference in extent metadata found!
158462023-12-10T18:56:02.588ZA Difference in extent metadata is expected here
158472023-12-10T18:56:02.588ZOn loop 64
158482023-12-10T18:56:02.588Z
158492023-12-10T18:56:02.588ZVerifying data now
158502023-12-10T18:56:09.136ZNo change after verify
158512023-12-10T18:56:09.136ZLoop: 64 Downstairs dump after verify (and repair):
158522023-12-10T18:56:09.143ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
158532023-12-10T18:56:09.143Z 0 000-019 127 127 127 221 221 221 F F F
158542023-12-10T18:56:09.143Z 1 020-039 126 126 126 220 220 220 F F F
158552023-12-10T18:56:09.144Z 2 040-059 127 127 127 221 221 221 F F F
158562023-12-10T18:56:09.144Z 3 060-079 128 128 128 223 223 223 F F F
158572023-12-10T18:56:09.144Z 4 080-099 128 128 128 223 223 223 F F F
158582023-12-10T18:56:09.144Z 5 100-119 126 126 126 220 220 220 F F F
158592023-12-10T18:56:09.144Z 6 120-139 128 128 128 224 224 224 F F F
158602023-12-10T18:56:09.144Z 7 140-159 128 128 128 223 223 223 F F F
158612023-12-10T18:56:09.144Z 8 160-179 119 119 119 208 208 208 F F F
158622023-12-10T18:56:09.144Z 9 180-199 125 125 125 219 219 219 F F F
158632023-12-10T18:56:09.144Z 10 200-219 112 112 112 197 197 197 F F F
158642023-12-10T18:56:09.144Z 11 220-239 129 129 129 225 225 225 F F F
158652023-12-10T18:56:09.144Z 12 240-259 126 126 126 220 220 220 F F F
158662023-12-10T18:56:09.144Z 13 260-279 126 126 126 220 220 220 F F F
158672023-12-10T18:56:09.144Z 14 280-299 124 124 124 218 218 218 F F F
158682023-12-10T18:56:09.144Z 15 300-319 127 127 127 221 221 221 F F F
158692023-12-10T18:56:09.144Z 16 320-339 128 128 128 223 223 223 F F F
158702023-12-10T18:56:09.144Z 17 340-359 122 122 122 214 214 214 F F F
158712023-12-10T18:56:09.144Z 18 360-379 122 122 122 214 214 214 F F F
158722023-12-10T18:56:09.145Z 19 380-399 127 127 127 221 221 221 F F F
158732023-12-10T18:56:09.145Z 20 400-419 129 129 129 225 225 225 F F F
158742023-12-10T18:56:09.145Z 21 420-439 108 108 108 189 189 189 F F F
158752023-12-10T18:56:09.145Z 22 440-459 122 122 122 214 214 214 F F F
158762023-12-10T18:56:09.145Z 23 460-479 123 123 123 215 215 215 F F F
158772023-12-10T18:56:09.145Z 24 480-499 129 129 129 225 225 225 F F F
158782023-12-10T18:56:09.145Z 25 500-519 128 128 128 223 223 223 F F F
158792023-12-10T18:56:09.145Z 26 520-539 129 129 129 225 225 225 F F F
158802023-12-10T18:56:09.145Z 27 540-559 129 129 129 225 225 225 F F F
158812023-12-10T18:56:09.145Z 28 560-579 126 126 126 220 220 220 F F F
158822023-12-10T18:56:09.145Z 29 580-599 128 128 128 223 223 223 F F F
158832023-12-10T18:56:09.145ZMax gen: 129, Max flush: 225
158842023-12-10T18:56:09.145Z
158852023-12-10T18:56:09.145ZBegin loop 65
158862023-12-10T18:56:09.145ZDownstairs to restart: 2
158872023-12-10T18:56:09.146Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.789413909Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682}
158882023-12-10T18:56:09.146Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.789890284Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682}
158892023-12-10T18:56:09.146Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.789919956Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682}
158902023-12-10T18:56:09.146Z{"msg":"Crucible da3d1143-f7f1-4a59-8dfc-5e7fa5c93a63 has session id: 8c875c49-29b7-45ea-a62b-9ccff494a9ab","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.789988671Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
158912023-12-10T18:56:09.146Z{"msg":"Upstairs opts: Upstairs UUID: da3d1143-f7f1-4a59-8dfc-5e7fa5c93a63, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","Crucible runtime is spawned
158922023-12-10T18:56:09.146Zv":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.790240576Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
158932023-12-10T18:56:09.146Z{"msg":"Crucible stats registered with UUID: da3d1143-f7f1-4a59-8dfc-5e7fa5c93a63","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.790343205Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
158942023-12-10T18:56:09.146Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30The guest has requested activation with gen:130
158952023-12-10T18:56:09.147Z,"time":"2023-12-10T18:56:08.790373583Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab","task":"up_listen"}
158962023-12-10T18:56:09.147Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.79050784Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
158972023-12-10T18:56:09.148Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.790534358Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
158982023-12-10T18:56:09.148Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.790597709Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"looper":"0","session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
158992023-12-10T18:56:09.148Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.790733771Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"looper":"1","session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159002023-12-10T18:56:09.148Z{"msg":"[0] da3d1143-f7f1-4a59-8dfc-5e7fa5c93a63 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.790816129Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"looper":"0","session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159012023-12-10T18:56:09.148Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.790890221Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"looper":"2","session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159022023-12-10T18:56:09.148Z{"msg":"[1] da3d1143-f7f1-4a59-8dfc-5e7fa5c93a63 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.790919329Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"looper":"1","session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159032023-12-10T18:56:09.149Z{"msg":"[2] da3d1143-f7f1-4a59-8dfc-5e7fa5c93a63 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.791054977Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"looper":"2","session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159042023-12-10T18:56:09.149Z{"msg":"da3d1143-f7f1-4a59-8dfc-5e7fa5c93a63 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.791364683Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159052023-12-10T18:56:09.149Z{"msg":"Set desired generation to :130","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.791417786Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159062023-12-10T18:56:09.149Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.791525767Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159072023-12-10T18:56:09.149Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.791840247Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159082023-12-10T18:56:09.149Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.791907397Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159092023-12-10T18:56:09.149Z{"msg":"[0] da3d1143-f7f1-4a59-8dfc-5e7fa5c93a63 (8c875c49-29b7-45ea-a62b-9ccff494a9ab) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.792218459Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159102023-12-10T18:56:09.149Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.792306169Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159112023-12-10T18:56:09.149Z{"msg":"[0] client is_active_req TRUE, promote! session 8c875c49-29b7-45ea-a62b-9ccff494a9ab","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.792350373Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159122023-12-10T18:56:09.150Z{"msg":"[1] da3d1143-f7f1-4a59-8dfc-5e7fa5c93a63 (8c875c49-29b7-45ea-a62b-9ccff494a9ab) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.792472448Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159132023-12-10T18:56:09.151Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.792649401Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159142023-12-10T18:56:09.151Z{"msg":"[1] client is_active_req TRUE, promote! session 8c875c49-29b7-45ea-a62b-9ccff494a9ab","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.792775658Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159152023-12-10T18:56:09.151Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.793108936Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159162023-12-10T18:56:09.151Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.793182984Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159172023-12-10T18:56:09.151Z{"msg":"da3d1143-f7f1-4a59-8dfc-5e7fa5c93a63 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.793229711Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159182023-12-10T18:56:09.151Z{"msg":"[2] da3d1143-f7f1-4a59-8dfc-5e7fa5c93a63 (8c875c49-29b7-45ea-a62b-9ccff494a9ab) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.79336089Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159192023-12-10T18:56:09.151Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.793422509Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159202023-12-10T18:56:09.151Z{"msg":"[2] client is_active_req TRUE, promote! session 8c875c49-29b7-45ea-a62b-9ccff494a9ab","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.793449688Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159212023-12-10T18:56:09.151Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.793518411Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159222023-12-10T18:56:09.151Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.793554015Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159232023-12-10T18:56:09.151Z{"msg":"da3d1143-f7f1-4a59-8dfc-5e7fa5c93a63 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.793607002Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159242023-12-10T18:56:09.152Z{"msg":"[0] da3d1143-f7f1-4a59-8dfc-5e7fa5c93a63 (8c875c49-29b7-45ea-a62b-9ccff494a9ab) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.793918927Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159252023-12-10T18:56:09.152Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.794001259Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159262023-12-10T18:56:09.152Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:08.79404583Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159272023-12-10T18:56:09.152Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.794122829Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159282023-12-10T18:56:09.152Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.794170648Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159292023-12-10T18:56:09.152Z{"msg":"da3d1143-f7f1-4a59-8dfc-5e7fa5c93a63 WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.7942425Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159302023-12-10T18:56:09.152Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.794341125Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159312023-12-10T18:56:09.152Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.794386383Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159322023-12-10T18:56:09.152Z{"msg":"[1] da3d1143-f7f1-4a59-8dfc-5e7fa5c93a63 (8c875c49-29b7-45ea-a62b-9ccff494a9ab) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.794456177Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159332023-12-10T18:56:09.152Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.794532465Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159342023-12-10T18:56:09.152Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:08.794576509Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159352023-12-10T18:56:09.152Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.794646223Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159362023-12-10T18:56:09.152Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.794700116Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159372023-12-10T18:56:09.152Z{"msg":"da3d1143-f7f1-4a59-8dfc-5e7fa5c93a63 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.794725089Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159382023-12-10T18:56:09.152Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.794766384Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159392023-12-10T18:56:09.153Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.794788806Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159402023-12-10T18:56:09.153Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.794830759Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159412023-12-10T18:56:09.153Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.794858605Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159422023-12-10T18:56:09.156Z{"msg":"da3d1143-f7f1-4a59-8dfc-5e7fa5c93a63 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.794900505Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159432023-12-10T18:56:09.156Z{"msg":"[2] da3d1143-f7f1-4a59-8dfc-5e7fa5c93a63 (8c875c49-29b7-45ea-a62b-9ccff494a9ab) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.795257185Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159442023-12-10T18:56:09.156Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.795338096Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159452023-12-10T18:56:09.156Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:08.795405091Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159462023-12-10T18:56:09.156Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.795431919Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159472023-12-10T18:56:09.156Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.795473699Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159482023-12-10T18:56:09.156Z{"msg":"da3d1143-f7f1-4a59-8dfc-5e7fa5c93a63 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.795497518Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159492023-12-10T18:56:09.156Z{"msg":"[0]R flush_numbers[0..12]: [221, 220, 221, 223, 223, 220, 224, 223, 208, 219, 197, 225]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.795539903Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":The guest has finished waiting for activation with:130
159502023-12-10T18:56:09.156Z1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159512023-12-10T18:56:09.156Z{"msg":"[0]R generation[0..12]: [127, 126, 127, 128, 128, 126, 128, 128, 119, 125, 112, 129]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.79564343Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159522023-12-10T18:56:09.198Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.795705716Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159532023-12-10T18:56:09.198Z{"msg":"[1]R flush_numbers[0..12]: [221, 220, 221, 223, 223, 220, 224, 223, 208, 219, 197, 225]","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO
159542023-12-10T18:56:09.198Z,"level":30,"time":"2023-12-10T18:56:08.795738855Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159552023-12-10T18:56:09.198Z{"msg":"[1]R generation[0..12]: [127, 126, 127, 128, 128, 126, 128, 128, 119, 125, 112, 129]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.795778569Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159562023-12-10T18:56:09.198Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.795818477Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159572023-12-10T18:56:09.198Z{"msg":"[2]R flush_numbers[0..12]: [221, 220, 221, 223, 223, 220, 224, 223, 208, 219, 197, 225]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.79586044Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159582023-12-10T18:56:09.199Z{"msg":"[2]R generation[0..12]: [127, 126, 127, 128, 128, 126, 128, 128, 119, 125, 112, 129]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.795934001Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159592023-12-10T18:56:09.199Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.796028908Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159602023-12-10T18:56:09.199Z{"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.796120626Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159612023-12-10T18:56:09.199Z{"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.796163879Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159622023-12-10T18:56:09.199Z{"msg":"Next flush: 226","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.796202297Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159632023-12-10T18:56:09.199Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.796234522Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159642023-12-10T18:56:09.199Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.796294843Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159652023-12-10T18:56:09.199Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.796335317Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159662023-12-10T18:56:09.199Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.796404478Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159672023-12-10T18:56:09.199Z{"msg":"da3d1143-f7f1-4a59-8dfc-5e7fa5c93a63 is now active with session: 8c875c49-29b7-45ea-a62b-9ccff494a9ab","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.796448785Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159682023-12-10T18:56:09.199Z{"msg":"da3d1143-f7f1-4a59-8dfc-5e7fa5c93a63 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.79650835Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159692023-12-10T18:56:09.199Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.796532961Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159702023-12-10T18:56:09.199Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.796555854Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159712023-12-10T18:56:09.200Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.796616378Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159722023-12-10T18:56:09.200Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.796641821Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159732023-12-10T18:56:09.200Z{Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
159742023-12-10T18:56:09.200Z"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.796757996Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159752023-12-10T18:56:09.200Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.796811336Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159762023-12-10T18:56:09.200Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.796835102Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159772023-12-10T18:56:09.201Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.796859437Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159782023-12-10T18:56:09.201Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.796912844Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159792023-12-10T18:56:09.201Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.796937236Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159802023-12-10T18:56:09.201Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:08.796962574Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1682,"session_id":"8c875c49-29b7-45ea-a62b-9ccff494a9ab"}
159812023-12-10T18:56:09.201ZLoading write count information from file "/tmp/test_repair_verify.data"
159822023-12-10T18:56:09.201ZRun Repair workload
159832023-12-10T18:56:09.201Z01/30 Write block 505 len 4096 data: 10
159842023-12-10T18:56:09.201Z02/30 Read block 432 len 24576
159852023-12-10T18:56:09.201Z03/30 Read block 231 len 24576
159862023-12-10T18:56:09.201Z04/30 Read block 133 len 32768
159872023-12-10T18:56:09.201Z05/30 Read block 15 len 4096
159882023-12-10T18:56:09.202Z06/30 Write block 426 len 32768 data: 9 12 10 9 9 8 7 7
159892023-12-10T18:56:09.202Z07/30 Write block 371 len 40960 data: 8 9 10 10 13 15 14 17 16 17
159902023-12-10T18:56:09.202Z08/30 Write block 449 len 24576 data: 9 9 10 11 11 9
159912023-12-10T18:56:09.202Z09/30 Read block 177 len 20480
159922023-12-10T18:56:09.202Z10/30 Read block 120 len 32768
159932023-12-10T18:56:09.202Z11/30 Write block 106 len 36864 data: 9 9 11 10 11 10 11 12 10
159942023-12-10T18:56:09.203Z12/30 Write block 130 len 12288 data: 9 8 5
159952023-12-10T18:56:09.203Z13/30 Write block 235 len 4096 data: 13
159962023-12-10T18:56:09.203Z14/30 Write block 162 len 40960 data: 16 16 15 14 14 14 13 11 10 12
159972023-12-10T18:56:09.203Z15/30 Read block 371 len 24576
159982023-12-10T18:56:09.203Z16/30 Write block 12 len 24576 data: 11 14 16 15 12 13
159992023-12-10T18:56:09.203Z17/30 Read block 59 len 28672
160002023-12-10T18:56:09.203Z18/30 Write block 541 len 40960 data: 17 17 15 14 10 10 9 10 8 7
160012023-12-10T18:56:09.203Z19/30 Read block 129 len 32768
160022023-12-10T18:56:09.203Z20/30 Read block 524 len 4096
160032023-12-10T18:56:09.203Z21/30 Read block 124 len 16384
160042023-12-10T18:56:09.203Z22/30 Read block 305 len 4096
160052023-12-10T18:56:09.203Z23/30 Read block 444 len 16384
160062023-12-10T18:56:09.203Z24/30 Write block 61 len 28672 data: 5 7 8 9 11 12 11
160072023-12-10T18:56:09.203Z25/30 Read block 380 len 40960
160082023-12-10T18:56:09.203Z26/30 Flush
160092023-12-10T18:56:09.203Z27/30 Write block 418 len 36864 data: 10 10 9 11 10 10 12 10 10
160102023-12-10T18:56:09.203Z28/30 Write block 400 len 28672 data: 10 11 11 12 10 12 11
160112023-12-10T18:56:09.203Z29/30 Read block 186 len 20480
160122023-12-10T18:56:09.204Z30/30 Write block 166 len 20480 data: 15 15 14 12 11
160132023-12-10T18:56:09.204Z----------------------------------------------------------------
160142023-12-10T18:56:09.204Z Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:30
160152023-12-10T18:56:09.204ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
160162023-12-10T18:56:09.204Z 1 Acked 1000 Write 1 Done Done Done false
160172023-12-10T18:56:09.204Z 2 Acked 1001 Read 6 Done Done Sent false
160182023-12-10T18:56:09.204Z 3 Acked 1002 Read 6 Done Done Sent false
160192023-12-10T18:56:09.204Z 4 Acked 1003 Read 8 Done Done Sent false
160202023-12-10T18:56:09.205Z 5 Acked 1004 Read 1 Done Done Sent false
160212023-12-10T18:56:09.205Z 6 Acked 1005 Write 8 Done Done Sent false
160222023-12-10T18:56:09.205Z 7 Acked 1006 Write 10 Done Done Sent false
160232023-12-10T18:56:09.205Z 8 Acked 1007 Write 6 Done Done Sent false
160242023-12-10T18:56:09.205Z 9 Acked 1008 Read 5 Done Done Sent false
160252023-12-10T18:56:09.205Z 10 Acked 1009 Read 8 Done Done Sent false
160262023-12-10T18:56:09.205Z 11 Acked 1010 Write 9 Done Done Sent false
160272023-12-10T18:56:09.205Z 12 Acked 1011 Write 3 Done Done Sent false
160282023-12-10T18:56:09.205Z 13 Acked 1012 Write 1 Done Done Sent false
160292023-12-10T18:56:09.205Z 14 Acked 1013 Write 10 Done Done Sent false
160302023-12-10T18:56:09.205Z 15 Acked 1014 Read 6 Done Done Sent false
160312023-12-10T18:56:09.205Z 16 Acked 1015 Write 6 Done Done Sent false
160322023-12-10T18:56:09.205Z 17 Acked 1016 Read 7 Done Done Sent false
160332023-12-10T18:56:09.205Z 18 Acked 1017 Write 10 Done Done Sent false
160342023-12-10T18:56:09.205Z 19 Acked 1018 Read 8 Done Done Sent false
160352023-12-10T18:56:09.205Z 20 Acked 1019 Read 1 Done Done Sent false
160362023-12-10T18:56:09.205Z 21 Acked 1020 Read 4 Done Done Sent false
160372023-12-10T18:56:09.206Z 22 Acked 1021 Read 1 Done Done Sent false
160382023-12-10T18:56:09.206Z 23 Acked 1022 Read 4 Done Done Sent false
160392023-12-10T18:56:09.206Z 24 Acked 1023 Write 7 Done Done Sent false
160402023-12-10T18:56:09.206Z 25 Acked 1024 Read 10 Done Done Sent false
160412023-12-10T18:56:09.206Z 26 Acked 1025 Flush 0 Done Done Sent false
160422023-12-10T18:56:09.206Z 27 Acked 1026 Write 9 Done Done Sent false
160432023-12-10T18:56:09.206Z 28 Acked 1027 Write 7 Done Done Sent false
160442023-12-10T18:56:09.206Z 29 Acked 1028 Read 5 Done Done Sent false
160452023-12-10T18:56:09.206Z 30 Acked 1029 Write 5 Sent Sent Sent false
160462023-12-10T18:56:09.207Z STATES DS:0 DS:1 DS:2 TOTAL
160472023-12-10T18:56:09.207Z New 0 0 0 0
160482023-12-10T18:56:09.207Z Sent 1 1 29 31
160492023-12-10T18:56:09.207Z Done 29 29 1 59
160502023-12-10T18:56:09.207Z Skipped 0 0 0 0
160512023-12-10T18:56:09.207Z Error 0 0 0 0
160522023-12-10T18:56:09.243ZLast Flush: 1025 1025 0
160532023-12-10T18:56:09.243ZDownstairs last five completed:
160542023-12-10T18:56:09.243ZUpstairs last five completed: 30 29 28 27 26 25
160552023-12-10T18:56:09.243ZWrote out file "/tmp/test_repair_verify.data"
160562023-12-10T18:56:09.243Z
160572023-12-10T18:56:09.243ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
160582023-12-10T18:56:09.244ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
160592023-12-10T18:56:09.244Z 0 000-019 130 130 127 226 226 221 F F F <---
160602023-12-10T18:56:09.244Z 1 020-039 126 126 126 220 220 220 F F F
160612023-12-10T18:56:09.244Z 2 040-059 127 127 127 221 221 221 F F F
160622023-12-10T18:56:09.244Z 3 060-079 130 130 128 226 226 223 F F F <---
160632023-12-10T18:56:09.244Z 4 080-099 128 128 128 223 223 223 F F F
160642023-12-10T18:56:09.244Z 5 100-119 130 130 126 226 226 220 F F F <---
160652023-12-10T18:56:09.244Z 6 120-139 130 130 128 226 226 224 F F F <---
160662023-12-10T18:56:09.244Z 7 140-159 128 128 128 223 223 223 F F F
160672023-12-10T18:56:09.244Z 8 160-179 130 130 119 226 226 208 T T F <---
160682023-12-10T18:56:09.244Z 9 180-199 125 125 125 219 219 219 F F F
160692023-12-10T18:56:09.244Z 10 200-219 112 112 112 197 197 197 F F F
160702023-12-10T18:56:09.244Z 11 220-239 130 130 129 226 226 225 F F F <---
160712023-12-10T18:56:09.244Z 12 240-259 126 126 126 220 220 220 F F F
160722023-12-10T18:56:09.245Z 13 260-279 126 126 126 220 220 220 F F F
160732023-12-10T18:56:09.245Z 14 280-299 124 124 124 218 218 218 F F F
160742023-12-10T18:56:09.245Z 15 300-319 127 127 127 221 221 221 F F F
160752023-12-10T18:56:09.245Z 16 320-339 128 128 128 223 223 223 F F F
160762023-12-10T18:56:09.245Z 17 340-359 122 122 122 214 214 214 F F F
160772023-12-10T18:56:09.245Z 18 360-379 130 130 122 226 226 214 F F F <---
160782023-12-10T18:56:09.245Z 19 380-399 130 130 127 226 226 221 F F F <---
160792023-12-10T18:56:09.245Z 20 400-419 129 129 129 225 225 225 T T F <---
160802023-12-10T18:56:09.245Z 21 420-439 130 130 108 226 226 189 T T F <---
160812023-12-10T18:56:09.246Z 22 440-459 130 130 122 226 226 214 F F F <---
160822023-12-10T18:56:09.246Z 23 460-479 123 123 123 215 215 215 F F F
160832023-12-10T18:56:09.246Z 24 480-499 129 129 129 225 225 225 F F F
160842023-12-10T18:56:09.246Z 25 500-519 130 130 128 226 226 223 F F T <---
160852023-12-10T18:56:09.247Z 26 520-539 129 129 129 225 225 225 F F F
160862023-12-10T18:56:09.247Z 27 540-559 130 130 129 226 226 225 F F F <---
160872023-12-10T18:56:09.247Z 28 560-579 126 126 126 220 220 220 F F F
160882023-12-10T18:56:09.248Z 29 580-599 128 128 128 223 223 223 F F F
160892023-12-10T18:56:09.248ZMax gen: 130, Max flush: 226
160902023-12-10T18:56:09.248ZError: Difference in extent metadata found!
160912023-12-10T18:56:09.248ZA Difference in extent metadata is expected here
160922023-12-10T18:56:09.248ZOn loop 65
160932023-12-10T18:56:09.248Z
160942023-12-10T18:56:09.248ZVerifying data now
160952023-12-10T18:56:14.145ZNo change after verify
160962023-12-10T18:56:14.145ZLoop: 65 Downstairs dump after verify (and repair):
160972023-12-10T18:56:14.170ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
160982023-12-10T18:56:14.170Z 0 000-019 130 130 130 226 226 226 F F F
160992023-12-10T18:56:14.170Z 1 020-039 126 126 126 220 220 220 F F F
161002023-12-10T18:56:14.170Z 2 040-059 127 127 127 221 221 221 F F F
161012023-12-10T18:56:14.179Z 3 060-079 130 130 130 226 226 226 F F F
161022023-12-10T18:56:14.179Z 4 080-099 128 128 128 223 223 223 F F F
161032023-12-10T18:56:14.179Z 5 100-119 130 130 130 226 226 226 F F F
161042023-12-10T18:56:14.179Z 6 120-139 130 130 130 226 226 226 F F F
161052023-12-10T18:56:14.179Z 7 140-159 128 128 128 223 223 223 F F F
161062023-12-10T18:56:14.179Z 8 160-179 131 131 131 227 227 227 F F F
161072023-12-10T18:56:14.179Z 9 180-199 125 125 125 219 219 219 F F F
161082023-12-10T18:56:14.180Z 10 200-219 112 112 112 197 197 197 F F F
161092023-12-10T18:56:14.180Z 11 220-239 130 130 130 226 226 226 F F F
161102023-12-10T18:56:14.180Z 12 240-259 126 126 126 220 220 220 F F F
161112023-12-10T18:56:14.180Z 13 260-279 126 126 126 220 220 220 F F F
161122023-12-10T18:56:14.180Z 14 280-299 124 124 124 218 218 218 F F F
161132023-12-10T18:56:14.180Z 15 300-319 127 127 127 221 221 221 F F F
161142023-12-10T18:56:14.180Z 16 320-339 128 128 128 223 223 223 F F F
161152023-12-10T18:56:14.180Z 17 340-359 122 122 122 214 214 214 F F F
161162023-12-10T18:56:14.180Z 18 360-379 130 130 130 226 226 226 F F F
161172023-12-10T18:56:14.180Z 19 380-399 130 130 130 226 226 226 F F F
161182023-12-10T18:56:14.180Z 20 400-419 131 131 131 227 227 227 F F F
161192023-12-10T18:56:14.181Z 21 420-439 131 131 131 227 227 227 F F F
161202023-12-10T18:56:14.181Z 22 440-459 130 130 130 226 226 226 F F F
161212023-12-10T18:56:14.181Z 23 460-479 123 123 123 215 215 215 F F F
161222023-12-10T18:56:14.181Z 24 480-499 129 129 129 225 225 225 F F F
161232023-12-10T18:56:14.181Z 25 500-519 130 130 130 226 226 226 F F F
161242023-12-10T18:56:14.181Z 26 520-539 129 129 129 225 225 225 F F F
161252023-12-10T18:56:14.181Z 27 540-559 130 130 130 226 226 226 F F F
161262023-12-10T18:56:14.181Z 28 560-579 126 126 126 220 220 220 F F F
161272023-12-10T18:56:14.181Z 29 580-599 128 128 128 223 223 223 F F F
161282023-12-10T18:56:14.181ZMax gen: 131, Max flush: 227
161292023-12-10T18:56:14.182Z
161302023-12-10T18:56:14.182ZBegin loop 66
161312023-12-10T18:56:14.182ZDownstairs to restart: 2
161322023-12-10T18:56:14.204Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.205371987Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690}
161332023-12-10T18:56:14.204Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.205810688Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690}
161342023-12-10T18:56:14.220Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.205875539Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690}
161352023-12-10T18:56:14.220Z{"msg":"Crucible 90034a55-d1ae-4bc2-b0e9-12ee34d15954 has session id: 7863d6ec-7229-4c1e-b32a-814dc5966b36","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.205942743Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161362023-12-10T18:56:14.220Z{"msg":"Upstairs opts: Upstairs UUID: 90034a55-d1ae-4bc2-b0e9-12ee34d15954, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.206065126Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161372023-12-10T18:56:14.220Z{"msg":"Crucible stats registered with UUID: 90034a55-d1ae-4bc2-b0e9-12ee34d15954"Crucible runtime is spawned
161382023-12-10T18:56:14.221Z,"v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.206403145Z","hostname":"The guest has requested activation with gen:132
161392023-12-10T18:56:14.221Zip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161402023-12-10T18:56:14.221Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.206841824Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"looper":"1","session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161412023-12-10T18:56:14.221Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.206925417Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"looper":"0","session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161422023-12-10T18:56:14.221Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.206965909Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"looper":"2","session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161432023-12-10T18:56:14.222Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.207032344Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36","task":"up_listen"}
161442023-12-10T18:56:14.222Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.207075431Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161452023-12-10T18:56:14.222Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.207142901Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161462023-12-10T18:56:14.222Z{"msg":"[1] 90034a55-d1ae-4bc2-b0e9-12ee34d15954 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.207182937Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"looper":"1","session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161472023-12-10T18:56:14.222Z{"msg":"[0] 90034a55-d1ae-4bc2-b0e9-12ee34d15954 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.207260375Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"looper":"0","session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161482023-12-10T18:56:14.223Z{"msg":"90034a55-d1ae-4bc2-b0e9-12ee34d15954 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.207302501Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161492023-12-10T18:56:14.223Z{"msg":"[2] 90034a55-d1ae-4bc2-b0e9-12ee34d15954 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.207378357Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"looper":"2","session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161502023-12-10T18:56:14.223Z{"msg":"Set desired generation to :132","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.207444479Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161512023-12-10T18:56:14.223Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.207486752Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161522023-12-10T18:56:14.224Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.207614549Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161532023-12-10T18:56:14.224Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.207640337Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161542023-12-10T18:56:14.224Z{"msg":"[0] 90034a55-d1ae-4bc2-b0e9-12ee34d15954 (7863d6ec-7229-4c1e-b32a-814dc5966b36) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.207874559Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161552023-12-10T18:56:14.224Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.207931594Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161562023-12-10T18:56:14.224Z{"msg":"[0] client is_active_req TRUE, promote! session 7863d6ec-7229-4c1e-b32a-814dc5966b36","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.207956562Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161572023-12-10T18:56:14.226Z{"msg":"[1] 90034a55-d1ae-4bc2-b0e9-12ee34d15954 (7863d6ec-7229-4c1e-b32a-814dc5966b36) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.208124386Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161582023-12-10T18:56:14.226Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.20820212Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161592023-12-10T18:56:14.226Z{"msg":"[1] client is_active_req TRUE, promote! session 7863d6ec-7229-4c1e-b32a-814dc5966b36","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.208230919Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161602023-12-10T18:56:14.227Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.208635572Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161612023-12-10T18:56:14.227Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.208694855Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161622023-12-10T18:56:14.227Z{"msg":"90034a55-d1ae-4bc2-b0e9-12ee34d15954 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.20871984Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161632023-12-10T18:56:14.227Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.208848977Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161642023-12-10T18:56:14.228Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.208910498Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161652023-12-10T18:56:14.228Z{"msg":"90034a55-d1ae-4bc2-b0e9-12ee34d15954 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.208935204Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161662023-12-10T18:56:14.228Z{"msg":"[0] 90034a55-d1ae-4bc2-b0e9-12ee34d15954 (7863d6ec-7229-4c1e-b32a-814dc5966b36) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.209476615Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161672023-12-10T18:56:14.228Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.209689353Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161682023-12-10T18:56:14.228Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:14.209789911Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161692023-12-10T18:56:14.228Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.209845688Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161702023-12-10T18:56:14.229Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.209870108Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161712023-12-10T18:56:14.229Z{"msg":"90034a55-d1ae-4bc2-b0e9-12ee34d15954 WaitQuorum WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.209926943Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161722023-12-10T18:56:14.229Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.209964919Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161732023-12-10T18:56:14.229Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.210010349Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161742023-12-10T18:56:14.230Z{"msg":"[1] 90034a55-d1ae-4bc2-b0e9-12ee34d15954 (7863d6ec-7229-4c1e-b32a-814dc5966b36) WaitQuorum WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.210036279Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161752023-12-10T18:56:14.230Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.210077424Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161762023-12-10T18:56:14.231Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:14.210099588Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161772023-12-10T18:56:14.231Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.210119416Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161782023-12-10T18:56:14.231Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.210159264Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161792023-12-10T18:56:14.231Z{"msg":"90034a55-d1ae-4bc2-b0e9-12ee34d15954 WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.210182049Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161802023-12-10T18:56:14.231Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.210227307Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161812023-12-10T18:56:14.232Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.210249845Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161822023-12-10T18:56:14.232Z{"msg":"[2] 90034a55-d1ae-4bc2-b0e9-12ee34d15954 (7863d6ec-7229-4c1e-b32a-814dc5966b36) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.210616041Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161832023-12-10T18:56:14.232Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.210672545Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161842023-12-10T18:56:14.232Z{"msg":"[2] client is_active_req TRUE, promote! session 7863d6ec-7229-4c1e-b32a-814dc5966b36","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.210697587Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161852023-12-10T18:56:14.232Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.211286223Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161862023-12-10T18:56:14.233Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.211357971Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161872023-12-10T18:56:14.234Z{"msg":"90034a55-d1ae-4bc2-b0e9-12ee34d15954 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.211383173Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161882023-12-10T18:56:14.234Z{"msg":"[2] 90034a55-d1ae-4bc2-b0e9-12ee34d15954 (7863d6ec-7229-4c1e-b32a-814dc5966b36) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.211908832Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161892023-12-10T18:56:14.234Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.211983668Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161902023-12-10T18:56:14.234Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:14.212026243Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161912023-12-10T18:56:14.234Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.212078563Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161922023-12-10T18:56:14.235Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.212102532Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161932023-12-10T18:56:14.235Z{"msg":"90034a55-d1ae-4bc2-b0e9-12ee34d15954 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.212145105Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161942023-12-10T18:56:14.235Z{"msg":"[0]R flush_numbers[0..12]: [226, 220, 221, 226, 223, 226, 226, 223, 227, 219, 197, 226]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.212168989Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161952023-12-10T18:56:14.235Z{"msg":"[0]R generation[0..12]: [130, 126, 127, 130, 128, 130, 130, 128, 131, 125, 112, 130]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.212216709Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161962023-12-10T18:56:14.235Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.212240713Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
161972023-12-10T18:56:14.236Z{"msg":"The guest has finished waiting for activation with:132
161982023-12-10T18:56:14.236Z[1]R flush_numbers[0..12]: [226, 220, 221, 226, 223, 226, 226, 223, 227, 219, 197, 226]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
161992023-12-10T18:56:14.236Z,"time":"2023-12-10T18:56:14.212265528Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
162002023-12-10T18:56:14.236Z{"msg":"[1]R generation[0..12]: [130, 126, 127, 130, 128, 130, 130, 128, 131, 125, 112, 130]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.212287646Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
162012023-12-10T18:56:14.236Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.212309669Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
162022023-12-10T18:56:14.237Z{"msg":"[2]R flush_numbers[0..12]: [226, 220, 221, 226, 223, 226, 226, 223, 227, 219, 197, 226]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.212365014Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
162032023-12-10T18:56:14.237Z{"msg":"[2]R generation[0..12]: [130, 126, 127, 130, 128, 130, 130, 128, 131, 125, 112, 130]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.212408483Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
162042023-12-10T18:56:14.237Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.212484606Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
162052023-12-10T18:56:14.237Z{"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.212525203Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
162062023-12-10T18:56:14.237Z{"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.21260725Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
162072023-12-10T18:56:14.239Z{"msg":"Next flush: 228","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.212639036Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
162082023-12-10T18:56:14.239Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.212695506Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
162092023-12-10T18:56:14.239Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.212734338Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
162102023-12-10T18:56:14.240Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.212793686Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
162112023-12-10T18:56:14.240Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.212835109Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
162122023-12-10T18:56:14.240Z{"msg":"90034a55-d1ae-4bc2-b0e9-12ee34d15954 is now active with session: 7863d6ec-7229-4c1e-b32a-814dc5966b36","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.212903439Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
162132023-12-10T18:56:14.240Z{"msg":"90034a55-d1ae-4bc2-b0e9-12ee34d15954 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.212945059Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
162142023-12-10T18:56:14.240Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.212983653Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
162152023-12-10T18:56:14.241Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.213019755Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
162162023-12-10T18:56:14.241Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.213117875Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
162172023-12-10T18:56:14.241Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.213158229Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
162182023-12-10T18:56:14.242Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.213182644Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
162192023-12-10T18:56:14.242Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.213240462Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
162202023-12-10T18:56:14.242Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.213277978Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
162212023-12-10T18:56:14.242Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.213310854Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
162222023-12-10T18:56:14.242ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
162232023-12-10T18:56:14.242Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.213376296Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
162242023-12-10T18:56:14.243Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.213400797Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
162252023-12-10T18:56:14.243Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:14.213420764Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1690,"session_id":"7863d6ec-7229-4c1e-b32a-814dc5966b36"}
162262023-12-10T18:56:14.243ZLoading write count information from file "/tmp/test_repair_verify.data"
162272023-12-10T18:56:14.243ZRun Repair workload
162282023-12-10T18:56:14.243Z01/30 Write block 61 len 28672 data: 6 8 9 10 12 13 12
162292023-12-10T18:56:14.243Z02/30 Write block 43 len 12288 data: 7 7 10
162302023-12-10T18:56:14.244Z03/30 Write block 60 len 28672 data: 6 7 9 10 11 13 14
162312023-12-10T18:56:14.244Z04/30 Read block 104 len 24576
162322023-12-10T18:56:14.253Z05/30 Write block 413 len 28672 data: 5 6 7 8 10 11 11
162332023-12-10T18:56:14.253Z06/30 Write block 529 len 4096 data: 9
162342023-12-10T18:56:14.254Z07/30 Write block 418 len 28672 data: 12 12 10 12 11 11 13
162352023-12-10T18:56:14.254Z08/30 Flush
162362023-12-10T18:56:14.255Z09/30 Write block 90 len 12288 data: 9 8 9
162372023-12-10T18:56:14.255Z10/30 Read block 433 len 24576
162382023-12-10T18:56:14.255Z11/30 Read block 455 len 36864
162392023-12-10T18:56:14.255Z12/30 Flush
162402023-12-10T18:56:14.255Z13/30 Write block 461 len 28672 data: 7 9 9 8 11 8 10
162412023-12-10T18:56:14.255Z14/30 Read block 153 len 4096
162422023-12-10T18:56:14.255Z15/30 Read block 308 len 36864
162432023-12-10T18:56:14.255Z16/30 Read block 375 len 20480
162442023-12-10T18:56:14.255Z17/30 Read block 323 len 28672
162452023-12-10T18:56:14.255Z18/30 Write block 86 len 16384 data: 16 13 14 12
162462023-12-10T18:56:14.255Z19/30 Write block 582 len 36864 data: 8 7 8 6 8 9 10 11 13
162472023-12-10T18:56:14.255Z20/30 Read block 267 len 32768
162482023-12-10T18:56:14.255Z21/30 Read block 415 len 28672
162492023-12-10T18:56:14.256Z22/30 Read block 444 len 32768
162502023-12-10T18:56:14.256Z23/30 Flush
162512023-12-10T18:56:14.263Z24/30 Write block 299 len 4096 data: 12
162522023-12-10T18:56:14.263Z25/30 Write block 148 len 32768 data: 8 9 8 7 7 7 12 7
162532023-12-10T18:56:14.263Z26/30 Write block 168 len 16384 data: 15 13 12 13
162542023-12-10T18:56:14.263Z27/30 Write block 174 len 36864 data: 10 8 8 9 7 5 5 7 9
162552023-12-10T18:56:14.264Z28/30 Write block 34 len 20480 data: 10 10 13 11 10
162562023-12-10T18:56:14.264Z29/30 Read block 25 len 40960
162572023-12-10T18:56:14.264Z30/30 Read block 331 len 32768
162582023-12-10T18:56:14.273Z----------------------------------------------------------------
162592023-12-10T18:56:14.274Z Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30
162602023-12-10T18:56:14.274ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
162612023-12-10T18:56:14.274Z 1 Acked 1000 Write 7 Done Done Done false
162622023-12-10T18:56:14.274Z 2 Acked 1001 Write 3 Done Done Done false
162632023-12-10T18:56:14.274Z 3 Acked 1002 Write 7 Done Done Sent false
162642023-12-10T18:56:14.274Z 4 Acked 1003 Read 6 Done Done Sent false
162652023-12-10T18:56:14.274Z 5 Acked 1004 Write 7 Done Done Sent false
162662023-12-10T18:56:14.274Z 6 Acked 1005 Write 1 Done Done Sent false
162672023-12-10T18:56:14.274Z 7 Acked 1006 Write 7 Done Done Sent false
162682023-12-10T18:56:14.274Z 8 Acked 1007 Flush 0 Done Done Sent false
162692023-12-10T18:56:14.274Z 9 Acked 1008 Write 3 Done Done Sent false
162702023-12-10T18:56:14.274Z 10 Acked 1009 Read 6 Done Done Sent false
162712023-12-10T18:56:14.274Z 11 Acked 1010 Read 9 Done Done Sent false
162722023-12-10T18:56:14.274Z 12 Acked 1011 Flush 0 Done Done Sent false
162732023-12-10T18:56:14.274Z 13 Acked 1012 Write 7 Done Done Sent false
162742023-12-10T18:56:14.274Z 14 Acked 1013 Read 1 Done Done Sent false
162752023-12-10T18:56:14.275Z 15 Acked 1014 Read 9 Done Done Sent false
162762023-12-10T18:56:14.275Z 16 Acked 1015 Read 5 Done Done Sent false
162772023-12-10T18:56:14.275Z 17 Acked 1016 Read 7 Done Done Sent false
162782023-12-10T18:56:14.275Z 18 Acked 1017 Write 4 Done Done Sent false
162792023-12-10T18:56:14.275Z 19 Acked 1018 Write 9 Done Done Sent false
162802023-12-10T18:56:14.275Z 20 Acked 1019 Read 8 Done Done Sent false
162812023-12-10T18:56:14.275Z 21 Acked 1020 Read 7 Done Done Sent false
162822023-12-10T18:56:14.275Z 22 Acked 1021 Read 8 Done Done Sent false
162832023-12-10T18:56:14.275Z 23 Acked 1022 Flush 0 Done Done Sent false
162842023-12-10T18:56:14.275Z 24 Acked 1023 Write 1 Done Done Sent false
162852023-12-10T18:56:14.275Z 25 Acked 1024 Write 8 Done Done Sent false
162862023-12-10T18:56:14.275Z 26 Acked 1025 Write 4 Done Done Sent false
162872023-12-10T18:56:14.275Z 27 Acked 1026 Write 9 Done Done Sent false
162882023-12-10T18:56:14.275Z 28 Acked 1027 Write 5 Done Done Sent false
162892023-12-10T18:56:14.275Z 29 Acked 1028 Read 10 Done Done Sent false
162902023-12-10T18:56:14.275Z 30 Acked 1029 Read 8 Done Done Sent false
162912023-12-10T18:56:14.275Z STATES DS:0 DS:1 DS:2 TOTAL
162922023-12-10T18:56:14.276Z New 0 0 0 0
162932023-12-10T18:56:14.276Z Sent 0 0 28 28
162942023-12-10T18:56:14.276Z Done 30 30 2 62
162952023-12-10T18:56:14.276Z Skipped 0 0 0 0
162962023-12-10T18:56:14.276Z Error 0 0 0 0
162972023-12-10T18:56:14.276ZLast Flush: 1022 1022 0
162982023-12-10T18:56:14.276ZDownstairs last five completed:
162992023-12-10T18:56:14.276ZUpstairs last five completed: 30 29 28 27 26 25
163002023-12-10T18:56:14.276ZWrote out file "/tmp/test_repair_verify.data"
163012023-12-10T18:56:14.281Z
163022023-12-10T18:56:14.281ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
163032023-12-10T18:56:14.302ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
163042023-12-10T18:56:14.303Z 0 000-019 130 130 130 226 226 226 F F F
163052023-12-10T18:56:14.303Z 1 020-039 126 126 126 220 220 220 T T F <---
163062023-12-10T18:56:14.303Z 2 040-059 132 132 127 228 228 221 F F T <---
163072023-12-10T18:56:14.304Z 3 060-079 132 132 130 228 228 226 F F T <---
163082023-12-10T18:56:14.308Z 4 080-099 132 132 128 230 230 223 F F F <---
163092023-12-10T18:56:14.309Z 5 100-119 130 130 130 226 226 226 F F F
163102023-12-10T18:56:14.309Z 6 120-139 130 130 130 226 226 226 F F F
163112023-12-10T18:56:14.309Z 7 140-159 128 128 128 223 223 223 T T F <---
163122023-12-10T18:56:14.309Z 8 160-179 131 131 131 227 227 227 T T F <---
163132023-12-10T18:56:14.309Z 9 180-199 125 125 125 219 219 219 T T F <---
163142023-12-10T18:56:14.309Z 10 200-219 112 112 112 197 197 197 F F F
163152023-12-10T18:56:14.309Z 11 220-239 130 130 130 226 226 226 F F F
163162023-12-10T18:56:14.309Z 12 240-259 126 126 126 220 220 220 F F F
163172023-12-10T18:56:14.309Z 13 260-279 126 126 126 220 220 220 F F F
163182023-12-10T18:56:14.309Z 14 280-299 124 124 124 218 218 218 T T F <---
163192023-12-10T18:56:14.309Z 15 300-319 127 127 127 221 221 221 F F F
163202023-12-10T18:56:14.309Z 16 320-339 128 128 128 223 223 223 F F F
163212023-12-10T18:56:14.309Z 17 340-359 122 122 122 214 214 214 F F F
163222023-12-10T18:56:14.309Z 18 360-379 130 130 130 226 226 226 F F F
163232023-12-10T18:56:14.309Z 19 380-399 130 130 130 226 226 226 F F F
163242023-12-10T18:56:14.309Z 20 400-419 132 132 131 228 228 227 F F F <---
163252023-12-10T18:56:14.309Z 21 420-439 132 132 131 228 228 227 F F F <---
163262023-12-10T18:56:14.310Z 22 440-459 130 130 130 226 226 226 F F F
163272023-12-10T18:56:14.310Z 23 460-479 132 132 123 230 230 215 F F F <---
163282023-12-10T18:56:14.310Z 24 480-499 129 129 129 225 225 225 F F F
163292023-12-10T18:56:14.310Z 25 500-519 130 130 130 226 226 226 F F F
163302023-12-10T18:56:14.310Z 26 520-539 132 132 129 228 228 225 F F F <---
163312023-12-10T18:56:14.310Z 27 540-559 130 130 130 226 226 226 F F F
163322023-12-10T18:56:14.310Z 28 560-579 126 126 126 220 220 220 F F F
163332023-12-10T18:56:14.311Z 29 580-599 132 132 128 230 230 223 F F F <---
163342023-12-10T18:56:14.311ZMax gen: 132, Max flush: 230
163352023-12-10T18:56:14.311ZError: Difference in extent metadata found!
163362023-12-10T18:56:14.311ZA Difference in extent metadata is expected here
163372023-12-10T18:56:14.311ZOn loop 66
163382023-12-10T18:56:14.311Z
163392023-12-10T18:56:14.314ZVerifying data now
163402023-12-10T18:56:20.561ZNo change after verify
163412023-12-10T18:56:20.562ZLoop: 66 Downstairs dump after verify (and repair):
163422023-12-10T18:56:20.642ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
163432023-12-10T18:56:20.642Z 0 000-019 130 130 130 226 226 226 F F F
163442023-12-10T18:56:20.653Z 1 020-039 133 133 133 231 231 231 F F F
163452023-12-10T18:56:20.653Z 2 040-059 132 132 132 228 228 228 F F F
163462023-12-10T18:56:20.653Z 3 060-079 132 132 132 228 228 228 F F F
163472023-12-10T18:56:20.653Z 4 080-099 132 132 132 230 230 230 F F F
163482023-12-10T18:56:20.653Z 5 100-119 130 130 130 226 226 226 F F F
163492023-12-10T18:56:20.653Z 6 120-139 130 130 130 226 226 226 F F F
163502023-12-10T18:56:20.653Z 7 140-159 133 133 133 231 231 231 F F F
163512023-12-10T18:56:20.653Z 8 160-179 133 133 133 231 231 231 F F F
163522023-12-10T18:56:20.653Z 9 180-199 133 133 133 231 231 231 F F F
163532023-12-10T18:56:20.653Z 10 200-219 112 112 112 197 197 197 F F F
163542023-12-10T18:56:20.653Z 11 220-239 130 130 130 226 226 226 F F F
163552023-12-10T18:56:20.653Z 12 240-259 126 126 126 220 220 220 F F F
163562023-12-10T18:56:20.653Z 13 260-279 126 126 126 220 220 220 F F F
163572023-12-10T18:56:20.653Z 14 280-299 133 133 133 231 231 231 F F F
163582023-12-10T18:56:20.654Z 15 300-319 127 127 127 221 221 221 F F F
163592023-12-10T18:56:20.654Z 16 320-339 128 128 128 223 223 223 F F F
163602023-12-10T18:56:20.654Z 17 340-359 122 122 122 214 214 214 F F F
163612023-12-10T18:56:20.654Z 18 360-379 130 130 130 226 226 226 F F F
163622023-12-10T18:56:20.654Z 19 380-399 130 130 130 226 226 226 F F F
163632023-12-10T18:56:20.654Z 20 400-419 132 132 132 228 228 228 F F F
163642023-12-10T18:56:20.654Z 21 420-439 132 132 132 228 228 228 F F F
163652023-12-10T18:56:20.654Z 22 440-459 130 130 130 226 226 226 F F F
163662023-12-10T18:56:20.655Z 23 460-479 132 132 132 230 230 230 F F F
163672023-12-10T18:56:20.655Z 24 480-499 129 129 129 225 225 225 F F F
163682023-12-10T18:56:20.655Z 25 500-519 130 130 130 226 226 226 F F F
163692023-12-10T18:56:20.655Z 26 520-539 132 132 132 228 228 228 F F F
163702023-12-10T18:56:20.655Z 27 540-559 130 130 130 226 226 226 F F F
163712023-12-10T18:56:20.655Z 28 560-579 126 126 126 220 220 220 F F F
163722023-12-10T18:56:20.655Z 29 580-599 132 132 132 230 230 230 F F F
163732023-12-10T18:56:20.655ZMax gen: 133, Max flush: 231
163742023-12-10T18:56:20.655Z
163752023-12-10T18:56:20.655ZBegin loop 67
163762023-12-10T18:56:20.655ZDownstairs to restart: 2
163772023-12-10T18:56:20.655Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.626426095Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698}
163782023-12-10T18:56:20.655Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.626839109Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698}
163792023-12-10T18:56:20.655Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.626868608Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698}
163802023-12-10T18:56:20.655Z{"msg":"Crucible d82475de-95de-496a-ac68-3fa242b2546f has session id: df325ae6-9d90-4c0e-b18a-cf85420b9a0d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.626913706Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
163812023-12-10T18:56:20.655Z{"msg":"Upstairs opts: Upstairs UUID: d82475de-95de-496a-ac68-3fa242b2546f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.626952624Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
163822023-12-10T18:56:20.655Z{"msg":"Crucible stats registered with UUID: d82475de-95de-496a-ac68-3fa242b2546f","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.626976681Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
163832023-12-10T18:56:20.655ZCrucible runtime is spawned
163842023-12-10T18:56:20.656ZThe guest has requested activation with gen:134
163852023-12-10T18:56:20.656Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.627233128Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d","task":"up_listen"}
163862023-12-10T18:56:20.656Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.627306662Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
163872023-12-10T18:56:20.656Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.627347898Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"looper":"1","session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
163882023-12-10T18:56:20.656Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.627421429Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
163892023-12-10T18:56:20.656Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.627585148Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"looper":"2","session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
163902023-12-10T18:56:20.656Z{"msg":"[1] d82475de-95de-496a-ac68-3fa242b2546f looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.62765046Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"looper":"1","session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
163912023-12-10T18:56:20.656Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.62776846Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"looper":"0","session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
163922023-12-10T18:56:20.656Z{"msg":"[2] d82475de-95de-496a-ac68-3fa242b2546f looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.627829605Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"looper":"2","session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
163932023-12-10T18:56:20.656Z{"msg":"[0] d82475de-95de-496a-ac68-3fa242b2546f looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.627890356Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"looper":"0","session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
163942023-12-10T18:56:20.656Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.628357394Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
163952023-12-10T18:56:20.656Z{"msg":"d82475de-95de-496a-ac68-3fa242b2546f active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.628419439Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
163962023-12-10T18:56:20.656Z{"msg":"Set desired generation to :134","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.628459248Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
163972023-12-10T18:56:20.656Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.628808941Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
163982023-12-10T18:56:20.656Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.628883822Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
163992023-12-10T18:56:20.656Z{"msg":"[1] d82475de-95de-496a-ac68-3fa242b2546f (df325ae6-9d90-4c0e-b18a-cf85420b9a0d) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.629130315Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164002023-12-10T18:56:20.656Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.629237384Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164012023-12-10T18:56:20.657Z{"msg":"[1] client is_active_req TRUE, promote! session df325ae6-9d90-4c0e-b18a-cf85420b9a0d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.629285761Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164022023-12-10T18:56:20.657Z{"msg":"[0] d82475de-95de-496a-ac68-3fa242b2546f (df325ae6-9d90-4c0e-b18a-cf85420b9a0d) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.629417668Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164032023-12-10T18:56:20.657Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.629489248Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164042023-12-10T18:56:20.657Z{"msg":"[0] client is_active_req TRUE, promote! session df325ae6-9d90-4c0e-b18a-cf85420b9a0d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.629534517Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164052023-12-10T18:56:20.657Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.629846794Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164062023-12-10T18:56:20.657Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.629938281Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164072023-12-10T18:56:20.657Z{"msg":"d82475de-95de-496a-ac68-3fa242b2546f WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.63004548Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164082023-12-10T18:56:20.657Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.630187669Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164092023-12-10T18:56:20.657Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.630250971Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164102023-12-10T18:56:20.657Z{"msg":"[2] d82475de-95de-496a-ac68-3fa242b2546f (df325ae6-9d90-4c0e-b18a-cf85420b9a0d) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.63028191Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164112023-12-10T18:56:20.657Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.630319197Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164122023-12-10T18:56:20.657Z{"msg":"[2] client is_active_req TRUE, promote! session df325ae6-9d90-4c0e-b18a-cf85420b9a0d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.63039597Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164132023-12-10T18:56:20.657Z{"msg":"d82475de-95de-496a-ac68-3fa242b2546f WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.630443987Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164142023-12-10T18:56:20.657Z{"msg":"[1] d82475de-95de-496a-ac68-3fa242b2546f (df325ae6-9d90-4c0e-b18a-cf85420b9a0d) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.630724642Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164152023-12-10T18:56:20.658Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.63077565Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164162023-12-10T18:56:20.658Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:20.630806552Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164172023-12-10T18:56:20.658Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.630827948Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164182023-12-10T18:56:20.658Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.630869653Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164192023-12-10T18:56:20.658Z{"msg":"d82475de-95de-496a-ac68-3fa242b2546f WaitActive WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.630946951Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164202023-12-10T18:56:20.658Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.630989159Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164212023-12-10T18:56:20.658Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.631049771Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164222023-12-10T18:56:20.658Z{"msg":"[0] d82475de-95de-496a-ac68-3fa242b2546f (df325ae6-9d90-4c0e-b18a-cf85420b9a0d) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.631077689Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164232023-12-10T18:56:20.658Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.63115528Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164242023-12-10T18:56:20.658Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:20.631197864Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164252023-12-10T18:56:20.658Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.631292322Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164262023-12-10T18:56:20.658Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.631321353Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164272023-12-10T18:56:20.658Z{"msg":"d82475de-95de-496a-ac68-3fa242b2546f WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.631342481Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164282023-12-10T18:56:20.658Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.631394144Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164292023-12-10T18:56:20.658Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.631416564Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164302023-12-10T18:56:20.659Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.631536523Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164312023-12-10T18:56:20.659Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.63161232Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164322023-12-10T18:56:20.659Z{"msg":"d82475de-95de-496a-ac68-3fa242b2546f WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.631692046Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164332023-12-10T18:56:20.659Z{"msg":"[2] d82475de-95de-496a-ac68-3fa242b2546f (df325ae6-9d90-4c0e-b18a-cf85420b9a0d) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.632251341Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164342023-12-10T18:56:20.660Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.632335123Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164352023-12-10T18:56:20.660Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:20.632378425Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164362023-12-10T18:56:20.660Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.632447373Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164372023-12-10T18:56:20.660Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.632516275Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164382023-12-10T18:56:20.660Z{"msg":"d82475de-95de-496a-ac68-3fa242b2546f WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.632583898Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164392023-12-10T18:56:20.660Z{"msg":"[0]R flush_numbers[0..12]: [226, 231, 228, 228, 230, 226, 226, 231, 231, 231, 197, 226]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.632656959Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"The guest has finished waiting for activation with:134
164402023-12-10T18:56:20.660ZWait for a query_work_queue command to finish before sending IO
164412023-12-10T18:56:20.660Z}
164422023-12-10T18:56:20.660Z{"msg":"[0]R generation[0..12]: [130, 133, 132, 132, 132, 130, 130, 133, 133, 133, 112, 130]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.632781126Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164432023-12-10T18:56:20.662Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.632865507Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164442023-12-10T18:56:20.736Z{"msg":"[1]R flush_numbers[0..12]: [226, 231, 228, 228, 230, 226, 226, 231, 231, 231, 197, 226]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.632962758Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164452023-12-10T18:56:20.737Z{"msg":"[1]R generation[0..12]: [130, 133, 132, 132, 132, 130, 130, 133, 133, 133, 112, 130]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.633005967Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164462023-12-10T18:56:20.737Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.63307598Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164472023-12-10T18:56:20.737Z{"msg":"[2]R flush_numbers[0..12]: [226, 231, 228, 228, 230, 226, 226, 231, 231, 231, 197, 226]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.633146864Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164482023-12-10T18:56:20.737Z{"msg":"[2]R generation[0..12]: [130, 133, 132, 132, 132, 130, 130, 133, 133, 133, 112, 130]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.633217535Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164492023-12-10T18:56:20.737Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.633258853Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164502023-12-10T18:56:20.737Z{"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.633325426Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164512023-12-10T18:56:20.737Z{"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.633398265Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164522023-12-10T18:56:20.737Z{"msg":"Next flush: 232","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.633439376Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164532023-12-10T18:56:20.737Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.633476086Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164542023-12-10T18:56:20.737Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.633550744Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164552023-12-10T18:56:20.737Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.633598663Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164562023-12-10T18:56:20.737Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.633686261Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164572023-12-10T18:56:20.737Z{"msg":"d82475de-95de-496a-ac68-3fa242b2546f is now active with session: df325ae6-9d90-4c0e-b18a-cf85420b9a0d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.633725008Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164582023-12-10T18:56:20.737Z{"msg":"d82475de-95de-496a-ac68-3fa242b2546f Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.633796961Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_idRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
164592023-12-10T18:56:20.737Z":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164602023-12-10T18:56:20.738Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.633927094Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164612023-12-10T18:56:20.738Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.633953543Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164622023-12-10T18:56:20.738Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.633994262Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164632023-12-10T18:56:20.738Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.63401654Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164642023-12-10T18:56:20.742Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.634056389Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164652023-12-10T18:56:20.742Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.63407822Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164662023-12-10T18:56:20.743Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.634117385Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164672023-12-10T18:56:20.743Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.634139234Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164682023-12-10T18:56:20.743Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.634178562Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164692023-12-10T18:56:20.743Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.634200253Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164702023-12-10T18:56:20.743Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:20.634239377Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1698,"session_id":"df325ae6-9d90-4c0e-b18a-cf85420b9a0d"}
164712023-12-10T18:56:20.743ZLoading write count information from file "/tmp/test_repair_verify.data"
164722023-12-10T18:56:20.743ZRun Repair workload
164732023-12-10T18:56:20.743Z01/30 Write block 346 len 36864 data: 10 12 11 14 12 13 15 14 13
164742023-12-10T18:56:20.743Z02/30 Read block 545 len 32768
164752023-12-10T18:56:20.743Z03/30 Write block 412 len 24576 data: 6 6 7 8 9 11
164762023-12-10T18:56:20.743Z04/30 Read block 63 len 20480
164772023-12-10T18:56:20.743Z05/30 Read block 89 len 20480
164782023-12-10T18:56:20.743Z06/30 Read block 373 len 40960
164792023-12-10T18:56:20.743Z07/30 Write block 126 len 40960 data: 15 14 15 13 10 9 6 6 5 5
164802023-12-10T18:56:20.743Z08/30 Read block 326 len 40960
164812023-12-10T18:56:20.743Z09/30 Read block 503 len 40960
164822023-12-10T18:56:20.743Z10/30 Read block 211 len 24576
164832023-12-10T18:56:20.743Z11/30 Read block 234 len 20480
164842023-12-10T18:56:20.744Z12/30 Read block 6 len 16384
164852023-12-10T18:56:20.744Z13/30 Flush
164862023-12-10T18:56:20.744Z14/30 Write block 348 len 20480 data: 12 15 13 14 16
164872023-12-10T18:56:20.744Z15/30 Read block 349 len 32768
164882023-12-10T18:56:20.744Z16/30 Read block 63 len 28672
164892023-12-10T18:56:20.744Z17/30 Read block 228 len 24576
164902023-12-10T18:56:20.744Z18/30 Read block 33 len 16384
164912023-12-10T18:56:20.775Z19/30 Write block 533 len 8192 data: 12 14
164922023-12-10T18:56:20.775Z20/30 Write block 458 len 12288 data: 11 9 8
164932023-12-10T18:56:20.775Z21/30 Read block 236 len 28672
164942023-12-10T18:56:20.776Z22/30 Read block 338 len 32768
164952023-12-10T18:56:20.776Z23/30 Flush
164962023-12-10T18:56:20.776Z24/30 Write block 451 len 4096 data: 11
164972023-12-10T18:56:20.776Z25/30 Read block 138 len 20480
164982023-12-10T18:56:20.776Z26/30 Read block 112 len 36864
164992023-12-10T18:56:20.776Z27/30 Write block 301 len 16384 data: 11 11 14 16
165002023-12-10T18:56:20.777Z28/30 Read block 37 len 24576
165012023-12-10T18:56:20.777Z29/30 Read block 54 len 24576
165022023-12-10T18:56:20.777Z30/30 Read block 111 len 36864
165032023-12-10T18:56:20.777Z----------------------------------------------------------------
165042023-12-10T18:56:20.777Z Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30
165052023-12-10T18:56:20.778ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
165062023-12-10T18:56:20.779Z 1 Acked 1000 Write 9 Done Done Done false
165072023-12-10T18:56:20.779Z 2 Acked 1001 Read 8 Done Done Done false
165082023-12-10T18:56:20.779Z 3 Acked 1002 Write 6 Done Done Done false
165092023-12-10T18:56:20.779Z 4 Acked 1003 Read 5 Done Done Sent false
165102023-12-10T18:56:20.779Z 5 Acked 1004 Read 5 Done Done Sent false
165112023-12-10T18:56:20.780Z 6 Acked 1005 Read 10 Done Done Sent false
165122023-12-10T18:56:20.780Z 7 Acked 1006 Write 10 Done Done Sent false
165132023-12-10T18:56:20.780Z 8 Acked 1007 Read 10 Done Done Sent false
165142023-12-10T18:56:20.780Z 9 Acked 1008 Read 10 Done Done Sent false
165152023-12-10T18:56:20.781Z 10 Acked 1009 Read 6 Done Done Sent false
165162023-12-10T18:56:20.781Z 11 Acked 1010 Read 5 Done Done Sent false
165172023-12-10T18:56:20.781Z 12 Acked 1011 Read 4 Done Done Sent false
165182023-12-10T18:56:20.781Z 13 Acked 1012 Flush 0 Done Done Sent false
165192023-12-10T18:56:20.781Z 14 Acked 1013 Write 5 Done Done Sent false
165202023-12-10T18:56:20.781Z 15 Acked 1014 Read 8 Done Done Sent false
165212023-12-10T18:56:20.782Z 16 Acked 1015 Read 7 Done Done Sent false
165222023-12-10T18:56:20.782Z 17 Acked 1016 Read 6 Done Done Sent false
165232023-12-10T18:56:20.782Z 18 Acked 1017 Read 4 Done Done Sent false
165242023-12-10T18:56:20.782Z 19 Acked 1018 Write 2 Done Done Sent false
165252023-12-10T18:56:20.782Z 20 Acked 1019 Write 3 Done Done Sent false
165262023-12-10T18:56:20.782Z 21 Acked 1020 Read 7 Done Done Sent false
165272023-12-10T18:56:20.782Z 22 Acked 1021 Read 8 Done Done Sent false
165282023-12-10T18:56:20.782Z 23 Acked 1022 Flush 0 Done Done Sent false
165292023-12-10T18:56:20.782Z 24 Acked 1023 Write 1 Done Done Sent false
165302023-12-10T18:56:20.782Z 25 Acked 1024 Read 5 Done Done Sent false
165312023-12-10T18:56:20.783Z 26 Acked 1025 Read 9 Done Done Sent false
165322023-12-10T18:56:20.783Z 27 Acked 1026 Write 4 Done Done Sent false
165332023-12-10T18:56:20.783Z 28 Acked 1027 Read 6 Done Done Sent false
165342023-12-10T18:56:20.783Z 29 Acked 1028 Read 6 Done Done Sent false
165352023-12-10T18:56:20.783Z 30 Acked 1029 Read 9 Done Sent Sent false
165362023-12-10T18:56:20.783Z STATES DS:0 DS:1 DS:2 TOTAL
165372023-12-10T18:56:20.784Z New 0 0 0 0
165382023-12-10T18:56:20.784Z Sent 0 1 27 28
165392023-12-10T18:56:20.784Z Done 30 29 3 62
165402023-12-10T18:56:20.784Z Skipped 0 0 0 0
165412023-12-10T18:56:20.784Z Error 0 0 0 0
165422023-12-10T18:56:20.784ZLast Flush: 1022 1022 0
165432023-12-10T18:56:20.785ZDownstairs last five completed:
165442023-12-10T18:56:20.785ZUpstairs last five completed: 30 29 28 27 26 25
165452023-12-10T18:56:20.785ZWrote out file "/tmp/test_repair_verify.data"
165462023-12-10T18:56:20.785Z
165472023-12-10T18:56:20.785ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
165482023-12-10T18:56:20.786ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
165492023-12-10T18:56:20.786Z 0 000-019 130 130 130 226 226 226 F F F
165502023-12-10T18:56:20.786Z 1 020-039 133 133 133 231 231 231 F F F
165512023-12-10T18:56:20.786Z 2 040-059 132 132 132 228 228 228 F F F
165522023-12-10T18:56:20.786Z 3 060-079 132 132 132 228 228 228 F F F
165532023-12-10T18:56:20.787Z 4 080-099 132 132 132 230 230 230 F F F
165542023-12-10T18:56:20.787Z 5 100-119 130 130 130 226 226 226 F F F
165552023-12-10T18:56:20.787Z 6 120-139 134 134 130 232 232 226 F F F <---
165562023-12-10T18:56:20.787Z 7 140-159 133 133 133 231 231 231 F F F
165572023-12-10T18:56:20.788Z 8 160-179 133 133 133 231 231 231 F F F
165582023-12-10T18:56:20.788Z 9 180-199 133 133 133 231 231 231 F F F
165592023-12-10T18:56:20.788Z 10 200-219 112 112 112 197 197 197 F F F
165602023-12-10T18:56:20.788Z 11 220-239 130 130 130 226 226 226 F F F
165612023-12-10T18:56:20.789Z 12 240-259 126 126 126 220 220 220 F F F
165622023-12-10T18:56:20.789Z 13 260-279 126 126 126 220 220 220 F F F
165632023-12-10T18:56:20.789Z 14 280-299 133 133 133 231 231 231 F F F
165642023-12-10T18:56:20.789Z 15 300-319 127 127 127 221 221 221 T T F <---
165652023-12-10T18:56:20.789Z 16 320-339 128 128 128 223 223 223 F F F
165662023-12-10T18:56:20.789Z 17 340-359 134 134 122 233 233 214 F F T <---
165672023-12-10T18:56:20.789Z 18 360-379 130 130 130 226 226 226 F F F
165682023-12-10T18:56:20.790Z 19 380-399 130 130 130 226 226 226 F F F
165692023-12-10T18:56:20.790Z 20 400-419 134 134 132 232 232 228 F F T <---
165702023-12-10T18:56:20.790Z 21 420-439 132 132 132 228 228 228 F F F
165712023-12-10T18:56:20.790Z 22 440-459 134 134 130 233 233 226 T T F <---
165722023-12-10T18:56:20.791Z 23 460-479 134 134 132 233 233 230 F F F <---
165732023-12-10T18:56:20.791Z 24 480-499 129 129 129 225 225 225 F F F
165742023-12-10T18:56:20.791Z 25 500-519 130 130 130 226 226 226 F F F
165752023-12-10T18:56:20.791Z 26 520-539 134 134 132 233 233 228 F F F <---
165762023-12-10T18:56:20.791Z 27 540-559 130 130 130 226 226 226 F F F
165772023-12-10T18:56:20.791Z 28 560-579 126 126 126 220 220 220 F F F
165782023-12-10T18:56:20.791Z 29 580-599 132 132 132 230 230 230 F F F
165792023-12-10T18:56:20.792ZMax gen: 134, Max flush: 233
165802023-12-10T18:56:20.792ZError: Difference in extent metadata found!
165812023-12-10T18:56:20.792ZA Difference in extent metadata is expected here
165822023-12-10T18:56:20.792ZOn loop 67
165832023-12-10T18:56:20.792Z
165842023-12-10T18:56:20.792ZVerifying data now
165852023-12-10T18:56:26.553ZNo change after verify
165862023-12-10T18:56:26.553ZLoop: 67 Downstairs dump after verify (and repair):
165872023-12-10T18:56:26.569ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
165882023-12-10T18:56:26.570Z 0 000-019 130 130 130 226 226 226 F F F
165892023-12-10T18:56:26.587Z 1 020-039 133 133 133 231 231 231 F F F
165902023-12-10T18:56:26.587Z 2 040-059 132 132 132 228 228 228 F F F
165912023-12-10T18:56:26.588Z 3 060-079 132 132 132 228 228 228 F F F
165922023-12-10T18:56:26.588Z 4 080-099 132 132 132 230 230 230 F F F
165932023-12-10T18:56:26.588Z 5 100-119 130 130 130 226 226 226 F F F
165942023-12-10T18:56:26.588Z 6 120-139 134 134 134 232 232 232 F F F
165952023-12-10T18:56:26.588Z 7 140-159 133 133 133 231 231 231 F F F
165962023-12-10T18:56:26.589Z 8 160-179 133 133 133 231 231 231 F F F
165972023-12-10T18:56:26.589Z 9 180-199 133 133 133 231 231 231 F F F
165982023-12-10T18:56:26.589Z 10 200-219 112 112 112 197 197 197 F F F
165992023-12-10T18:56:26.589Z 11 220-239 130 130 130 226 226 226 F F F
166002023-12-10T18:56:26.590Z 12 240-259 126 126 126 220 220 220 F F F
166012023-12-10T18:56:26.590Z 13 260-279 126 126 126 220 220 220 F F F
166022023-12-10T18:56:26.590Z 14 280-299 133 133 133 231 231 231 F F F
166032023-12-10T18:56:26.591Z 15 300-319 135 135 135 234 234 234 F F F
166042023-12-10T18:56:26.591Z 16 320-339 128 128 128 223 223 223 F F F
166052023-12-10T18:56:26.591Z 17 340-359 134 134 134 233 233 233 F F F
166062023-12-10T18:56:26.591Z 18 360-379 130 130 130 226 226 226 F F F
166072023-12-10T18:56:26.592Z 19 380-399 130 130 130 226 226 226 F F F
166082023-12-10T18:56:26.592Z 20 400-419 134 134 134 232 232 232 F F F
166092023-12-10T18:56:26.592Z 21 420-439 132 132 132 228 228 228 F F F
166102023-12-10T18:56:26.592Z 22 440-459 135 135 135 234 234 234 F F F
166112023-12-10T18:56:26.593Z 23 460-479 134 134 134 233 233 233 F F F
166122023-12-10T18:56:26.597Z 24 480-499 129 129 129 225 225 225 F F F
166132023-12-10T18:56:26.597Z 25 500-519 130 130 130 226 226 226 F F F
166142023-12-10T18:56:26.597Z 26 520-539 134 134 134 233 233 233 F F F
166152023-12-10T18:56:26.598Z 27 540-559 130 130 130 226 226 226 F F F
166162023-12-10T18:56:26.598Z 28 560-579 126 126 126 220 220 220 F F F
166172023-12-10T18:56:26.598Z 29 580-599 132 132 132 230 230 230 F F F
166182023-12-10T18:56:26.598ZMax gen: 135, Max flush: 234
166192023-12-10T18:56:26.599Z
166202023-12-10T18:56:26.599ZBegin loop 68
166212023-12-10T18:56:26.599ZDownstairs to restart: 2
166222023-12-10T18:56:26.619Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.610535481Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706}
166232023-12-10T18:56:26.619Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.610975906Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706}
166242023-12-10T18:56:26.619Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.611005447Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706}
166252023-12-10T18:56:26.619Z{"msg":"Crucible 9a6123b1-3392-41d2-ae85-d86e3b39ee96 has session id: ee0ae068-9b3f-48d0-9ec8-42062e7768a3","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.611073123Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166262023-12-10T18:56:26.620Z{"msg":"Upstairs opts: Upstairs UUID: 9a6123b1-3392-41d2-ae85-d86e3b39ee96, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.61111938Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166272023-12-10T18:56:26.620Z{"msg":"Crucible stats registered with UUID: 9a6123b1-3392-41d2-ae85-d86e3b39ee96","v":0,"name":"crucible","level":30Crucible runtime is spawned
166282023-12-10T18:56:26.620Z,"time":"2023-12-10T18:56:26.611184458Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166292023-12-10T18:56:26.620ZThe guest has requested activation with gen:136
166302023-12-10T18:56:26.620Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.611388782Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"looper":"1","session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166312023-12-10T18:56:26.620Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.611457391Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3","task":"up_listen"}
166322023-12-10T18:56:26.621Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.611484602Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166332023-12-10T18:56:26.621Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.611515826Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166342023-12-10T18:56:26.621Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.611710946Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"looper":"2","session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166352023-12-10T18:56:26.621Z{"msg":"[1] 9a6123b1-3392-41d2-ae85-d86e3b39ee96 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.611797817Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"looper":"1","session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166362023-12-10T18:56:26.621Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.611832113Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"looper":"0","session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166372023-12-10T18:56:26.621Z{"msg":"[2] 9a6123b1-3392-41d2-ae85-d86e3b39ee96 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.61185674Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"looper":"2","session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166382023-12-10T18:56:26.622Z{"msg":"[0] 9a6123b1-3392-41d2-ae85-d86e3b39ee96 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.612080032Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"looper":"0","session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166392023-12-10T18:56:26.622Z{"msg":"9a6123b1-3392-41d2-ae85-d86e3b39ee96 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.612435756Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166402023-12-10T18:56:26.623Z{"msg":"Set desired generation to :136","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.612556171Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166412023-12-10T18:56:26.623Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.612643834Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166422023-12-10T18:56:26.623Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.612918709Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166432023-12-10T18:56:26.623Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.613021733Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166442023-12-10T18:56:26.623Z{"msg":"[1] 9a6123b1-3392-41d2-ae85-d86e3b39ee96 (ee0ae068-9b3f-48d0-9ec8-42062e7768a3) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.613379821Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166452023-12-10T18:56:26.623Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.613574958Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166462023-12-10T18:56:26.624Z{"msg":"[1] client is_active_req TRUE, promote! session ee0ae068-9b3f-48d0-9ec8-42062e7768a3","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.613732918Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166472023-12-10T18:56:26.624Z{"msg":"[0] 9a6123b1-3392-41d2-ae85-d86e3b39ee96 (ee0ae068-9b3f-48d0-9ec8-42062e7768a3) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.613821765Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166482023-12-10T18:56:26.625Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.613938446Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166492023-12-10T18:56:26.625Z{"msg":"[0] client is_active_req TRUE, promote! session ee0ae068-9b3f-48d0-9ec8-42062e7768a3","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.613963922Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166502023-12-10T18:56:26.625Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.613990001Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166512023-12-10T18:56:26.626Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.614062689Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166522023-12-10T18:56:26.626Z{"msg":"9a6123b1-3392-41d2-ae85-d86e3b39ee96 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.614158328Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166532023-12-10T18:56:26.626Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.614201649Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166542023-12-10T18:56:26.627Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.614267187Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166552023-12-10T18:56:26.627Z{"msg":"9a6123b1-3392-41d2-ae85-d86e3b39ee96 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.614308907Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166562023-12-10T18:56:26.627Z{"msg":"[2] 9a6123b1-3392-41d2-ae85-d86e3b39ee96 (ee0ae068-9b3f-48d0-9ec8-42062e7768a3) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.614379474Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166572023-12-10T18:56:26.648Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.614419938Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166582023-12-10T18:56:26.648Z{"msg":"[2] client is_active_req TRUE, promote! session ee0ae068-9b3f-48d0-9ec8-42062e7768a3","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.614489472Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166592023-12-10T18:56:26.648Z{"msg":"[1] 9a6123b1-3392-41d2-ae85-d86e3b39ee96 (ee0ae068-9b3f-48d0-9ec8-42062e7768a3) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.614859828Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166602023-12-10T18:56:26.648Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.61496448Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166612023-12-10T18:56:26.649Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:26.615039612Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166622023-12-10T18:56:26.649Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.615083148Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166632023-12-10T18:56:26.649Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.615107091Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166642023-12-10T18:56:26.649Z{"msg":"9a6123b1-3392-41d2-ae85-d86e3b39ee96 WaitActive WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.615129005Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166652023-12-10T18:56:26.650Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.615169957Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166662023-12-10T18:56:26.650Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.615192494Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166672023-12-10T18:56:26.650Z{"msg":"[0] 9a6123b1-3392-41d2-ae85-d86e3b39ee96 (ee0ae068-9b3f-48d0-9ec8-42062e7768a3) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.615217062Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166682023-12-10T18:56:26.651Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.61525771Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166692023-12-10T18:56:26.651Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:26.615280531Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166702023-12-10T18:56:26.651Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.615300612Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166712023-12-10T18:56:26.651Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.615321615Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166722023-12-10T18:56:26.652Z{"msg":"9a6123b1-3392-41d2-ae85-d86e3b39ee96 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.615362793Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166732023-12-10T18:56:26.652Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.615385572Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166742023-12-10T18:56:26.652Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.615405446Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166752023-12-10T18:56:26.653Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.615427121Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166762023-12-10T18:56:26.653Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.615472213Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166772023-12-10T18:56:26.653Z{"msg":"9a6123b1-3392-41d2-ae85-d86e3b39ee96 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.615495891Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166782023-12-10T18:56:26.653Z{"msg":"[2] 9a6123b1-3392-41d2-ae85-d86e3b39ee96 (ee0ae068-9b3f-48d0-9ec8-42062e7768a3) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.616074503Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166792023-12-10T18:56:26.653Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.616140026Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166802023-12-10T18:56:26.654Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:26.616164458Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166812023-12-10T18:56:26.654Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.61618434Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166822023-12-10T18:56:26.654Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.616205158Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166832023-12-10T18:56:26.654Z{"msg":"9a6123b1-3392-41d2-ae85-d86e3b39ee96 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.616247559Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166842023-12-10T18:56:26.654Z{"msg":"[0]R flush_numbers[0..12]: [226, 231, 228, 228, 230, 226, 232, 231, 231, 231, 197, 226]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.616272115Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166852023-12-10T18:56:26.654Z{"msg":"[0]R generation[0..12]: [130, 133, 132, 132, 132, 130, 134, 133, 133, 133, 112, 130]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.616314625Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166862023-12-10T18:56:26.655Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.616339381Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166872023-12-10T18:56:26.655Z{"msg":"[1]R flush_numbers[0..12]: [226, 231, 228, 228, 230, 226, 232, 231, 231, 231, 197, 226]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.616381779Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166882023-12-10T18:56:26.655Z{"msg":"[1]R generation[0..12]: [130, 133, 132, 132, 132, 130, 134, 133, 133, 133, 112, 130]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.6164055Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166892023-12-10T18:56:26.656Z{"msg":"The guest has finished waiting for activation with:136
166902023-12-10T18:56:26.656Z[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
166912023-12-10T18:56:26.656Z,"time":"2023-12-10T18:56:26.616449799Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166922023-12-10T18:56:26.656Z{"msg":"[2]R flush_numbers[0..12]: [226, 231, 228, 228, 230, 226, 232, 231, 231, 231, 197, 226]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.616473961Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166932023-12-10T18:56:26.657Z{"msg":"[2]R generation[0..12]: [130, 133, 132, 132, 132, 130, 134, 133, 133, 133, 112, 130]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.61649545Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166942023-12-10T18:56:26.657Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.616517018Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166952023-12-10T18:56:26.657Z{"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.616536445Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166962023-12-10T18:56:26.657Z{"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.616556644Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166972023-12-10T18:56:26.657Z{"msg":"Next flush: 235","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.616575978Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166982023-12-10T18:56:26.657Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.616595506Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
166992023-12-10T18:56:26.658Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.616615243Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
167002023-12-10T18:56:26.659Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.616635414Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
167012023-12-10T18:56:26.659Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.616663355Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
167022023-12-10T18:56:26.659Z{"msg":"9a6123b1-3392-41d2-ae85-d86e3b39ee96 is now active with session: ee0ae068-9b3f-48d0-9ec8-42062e7768a3","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.616702156Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
167032023-12-10T18:56:26.659Z{"msg":"9a6123b1-3392-41d2-ae85-d86e3b39ee96 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.616776975Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
167042023-12-10T18:56:26.659Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.616820678Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
167052023-12-10T18:56:26.659Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.616889917Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
167062023-12-10T18:56:26.660Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.616966852Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
167072023-12-10T18:56:26.660Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.617009909Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
167082023-12-10T18:56:26.660Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.617078893Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
167092023-12-10T18:56:26.660Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.617148587Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
167102023-12-10T18:56:26.660Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.617189424Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
167112023-12-10T18:56:26.661Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
167122023-12-10T18:56:26.661Z,"time":"2023-12-10T18:56:26.617259121Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
167132023-12-10T18:56:26.661Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.617331656Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
167142023-12-10T18:56:26.661Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.617358576Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
167152023-12-10T18:56:26.662Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:26.617399945Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1706,"session_id":"ee0ae068-9b3f-48d0-9ec8-42062e7768a3"}
167162023-12-10T18:56:26.662ZLoading write count information from file "/tmp/test_repair_verify.data"
167172023-12-10T18:56:26.662ZRun Repair workload
167182023-12-10T18:56:26.662Z01/30 Write block 7 len 24576 data: 7 9 9 11 9 12
167192023-12-10T18:56:26.662Z02/30 Write block 230 len 28672 data: 18 17 15 12 15 14 13
167202023-12-10T18:56:26.662Z03/30 Write block 260 len 16384 data: 7 8 11 12
167212023-12-10T18:56:26.662Z04/30 Write block 347 len 16384 data: 13 13 16 14
167222023-12-10T18:56:26.663Z05/30 Write block 438 len 28672 data: 6 6 8 8 6 8 9
167232023-12-10T18:56:26.663Z06/30 Read block 32 len 40960
167242023-12-10T18:56:26.663Z07/30 Write block 7 len 40960 data: 8 10 10 12 10 13 15 17 16 13
167252023-12-10T18:56:26.663Z08/30 Write block 147 len 36864 data: 6 9 10 9 8 8 8 13 8
167262023-12-10T18:56:26.663Z09/30 Write block 138 len 20480 data: 5 7 6 5 5
167272023-12-10T18:56:26.663Z10/30 Write block 216 len 32768 data: 14 15 14 13 13 16 15 12
167282023-12-10T18:56:26.664Z11/30 Read block 417 len 28672
167292023-12-10T18:56:26.664Z12/30 Write block 189 len 36864 data: 7 10 11 11 11 11 15 15 17
167302023-12-10T18:56:26.664Z13/30 Read block 176 len 36864
167312023-12-10T18:56:26.687Z14/30 Read block 7 len 16384
167322023-12-10T18:56:26.688Z15/30 Write block 220 len 28672 data: 14 17 16 13 13 11 14
167332023-12-10T18:56:26.688Z16/30 Flush
167342023-12-10T18:56:26.688Z17/30 Write block 243 len 16384 data: 13 14 13 14
167352023-12-10T18:56:26.688Z18/30 Write block 81 len 40960 data: 16 17 16 16 14 17 14 15 13 10
167362023-12-10T18:56:26.688Z19/30 Read block 392 len 28672
167372023-12-10T18:56:26.688Z20/30 Read block 29 len 32768
167382023-12-10T18:56:26.688Z21/30 Read block 268 len 4096
167392023-12-10T18:56:26.689Z22/30 Write block 522 len 36864 data: 7 8 9 8 10 9 8 10 9
167402023-12-10T18:56:26.689Z23/30 Write block 329 len 8192 data: 10 11
167412023-12-10T18:56:26.689Z24/30 Read block 166 len 12288
167422023-12-10T18:56:26.689Z25/30 Write block 374 len 32768 data: 11 14 16 15 18 17 18 13
167432023-12-10T18:56:26.689Z26/30 Read block 542 len 24576
167442023-12-10T18:56:26.689Z27/30 Read block 563 len 8192
167452023-12-10T18:56:26.690Z28/30 Write block 182 len 8192 data: 10 10
167462023-12-10T18:56:26.690Z29/30 Write block 480 len 32768 data: 6 6 8 9 9 9 11 11
167472023-12-10T18:56:26.690Z30/30 Read block 147 len 12288
167482023-12-10T18:56:26.690Z----------------------------------------------------------------
167492023-12-10T18:56:26.690Z Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:30
167502023-12-10T18:56:26.690ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
167512023-12-10T18:56:26.691Z 1 Acked 1000 Write 6 Done Done Done false
167522023-12-10T18:56:26.691Z 2 Acked 1001 Write 7 Done Done Sent false
167532023-12-10T18:56:26.691Z 3 Acked 1002 Write 4 Done Done Sent false
167542023-12-10T18:56:26.691Z 4 Acked 1003 Write 4 Done Done Sent false
167552023-12-10T18:56:26.691Z 5 Acked 1004 Write 7 Done Done Sent false
167562023-12-10T18:56:26.691Z 6 Acked 1005 Read 10 Done Done Sent false
167572023-12-10T18:56:26.692Z 7 Acked 1006 Write 10 Done Done Sent false
167582023-12-10T18:56:26.692Z 8 Acked 1007 Write 9 Done Done Sent false
167592023-12-10T18:56:26.692Z 9 Acked 1008 Write 5 Done Done Sent false
167602023-12-10T18:56:26.692Z 10 Acked 1009 Write 8 Done Done Sent false
167612023-12-10T18:56:26.692Z 11 Acked 1010 Read 7 Done Done Sent false
167622023-12-10T18:56:26.692Z 12 Acked 1011 Write 9 Done Done Sent false
167632023-12-10T18:56:26.693Z 13 Acked 1012 Read 9 Done Done Sent false
167642023-12-10T18:56:26.693Z 14 Acked 1013 Read 4 Done Done Sent false
167652023-12-10T18:56:26.693Z 15 Acked 1014 Write 7 Done Done Sent false
167662023-12-10T18:56:26.693Z 16 Acked 1015 Flush 0 Done Done Sent false
167672023-12-10T18:56:26.693Z 17 Acked 1016 Write 4 Done Done Sent false
167682023-12-10T18:56:26.693Z 18 Acked 1017 Write 10 Done Done Sent false
167692023-12-10T18:56:26.694Z 19 Acked 1018 Read 7 Done Done Sent false
167702023-12-10T18:56:26.694Z 20 Acked 1019 Read 8 Done Done Sent false
167712023-12-10T18:56:26.694Z 21 Acked 1020 Read 1 Done Done Sent false
167722023-12-10T18:56:26.694Z 22 Acked 1021 Write 9 Done Done Sent false
167732023-12-10T18:56:26.694Z 23 Acked 1022 Write 2 Done Done Sent false
167742023-12-10T18:56:26.694Z 24 Acked 1023 Read 3 Done Done Sent false
167752023-12-10T18:56:26.695Z 25 Acked 1024 Write 8 Done Done Sent false
167762023-12-10T18:56:26.695Z 26 Acked 1025 Read 6 Done Done Sent false
167772023-12-10T18:56:26.695Z 27 Acked 1026 Read 2 Done Done Sent false
167782023-12-10T18:56:26.696Z 28 Acked 1027 Write 2 Done Done Sent false
167792023-12-10T18:56:26.696Z 29 Acked 1028 Write 8 Done Done Sent false
167802023-12-10T18:56:26.696Z 30 Acked 1029 Read 3 Sent Done Sent false
167812023-12-10T18:56:26.697Z STATES DS:0 DS:1 DS:2 TOTAL
167822023-12-10T18:56:26.697Z New 0 0 0 0
167832023-12-10T18:56:26.697Z Sent 1 0 29 30
167842023-12-10T18:56:26.697Z Done 29 30 1 60
167852023-12-10T18:56:26.697Z Skipped 0 0 0 0
167862023-12-10T18:56:26.697Z Error 0 0 0 0
167872023-12-10T18:56:26.698ZLast Flush: 1015 1015 0
167882023-12-10T18:56:26.698ZDownstairs last five completed:
167892023-12-10T18:56:26.698ZUpstairs last five completed: 30 29 28 27 26 25
167902023-12-10T18:56:26.698ZWrote out file "/tmp/test_repair_verify.data"
167912023-12-10T18:56:26.707Z
167922023-12-10T18:56:26.707ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
167932023-12-10T18:56:26.719ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
167942023-12-10T18:56:26.719Z 0 000-019 136 136 130 235 235 226 F F T <---
167952023-12-10T18:56:26.719Z 1 020-039 133 133 133 231 231 231 F F F
167962023-12-10T18:56:26.720Z 2 040-059 132 132 132 228 228 228 F F F
167972023-12-10T18:56:26.720Z 3 060-079 132 132 132 228 228 228 F F F
167982023-12-10T18:56:26.720Z 4 080-099 132 132 132 230 230 230 T T F <---
167992023-12-10T18:56:26.720Z 5 100-119 130 130 130 226 226 226 F F F
168002023-12-10T18:56:26.720Z 6 120-139 136 136 134 235 235 232 F F F <---
168012023-12-10T18:56:26.720Z 7 140-159 136 136 133 235 235 231 F F F <---
168022023-12-10T18:56:26.721Z 8 160-179 133 133 133 231 231 231 F F F
168032023-12-10T18:56:26.721Z 9 180-199 136 136 133 235 235 231 T T F <---
168042023-12-10T18:56:26.722Z 10 200-219 136 136 112 235 235 197 F F F <---
168052023-12-10T18:56:26.722Z 11 220-239 136 136 130 235 235 226 F F F <---
168062023-12-10T18:56:26.722Z 12 240-259 126 126 126 220 220 220 T T F <---
168072023-12-10T18:56:26.722Z 13 260-279 136 136 126 235 235 220 F F F <---
168082023-12-10T18:56:26.722Z 14 280-299 133 133 133 231 231 231 F F F
168092023-12-10T18:56:26.723Z 15 300-319 135 135 135 234 234 234 F F F
168102023-12-10T18:56:26.723Z 16 320-339 128 128 128 223 223 223 T T F <---
168112023-12-10T18:56:26.723Z 17 340-359 136 136 134 235 235 233 F F F <---
168122023-12-10T18:56:26.723Z 18 360-379 130 130 130 226 226 226 T T F <---
168132023-12-10T18:56:26.723Z 19 380-399 130 130 130 226 226 226 T T F <---
168142023-12-10T18:56:26.724Z 20 400-419 134 134 134 232 232 232 F F F
168152023-12-10T18:56:26.724Z 21 420-439 136 136 132 235 235 228 F F F <---
168162023-12-10T18:56:26.724Z 22 440-459 136 136 135 235 235 234 F F F <---
168172023-12-10T18:56:26.724Z 23 460-479 134 134 134 233 233 233 F F F
168182023-12-10T18:56:26.725Z 24 480-499 129 129 129 225 225 225 T T F <---
168192023-12-10T18:56:26.725Z 25 500-519 130 130 130 226 226 226 F F F
168202023-12-10T18:56:26.725Z 26 520-539 134 134 134 233 233 233 T T F <---
168212023-12-10T18:56:26.725Z 27 540-559 130 130 130 226 226 226 F F F
168222023-12-10T18:56:26.726Z 28 560-579 126 126 126 220 220 220 F F F
168232023-12-10T18:56:26.726Z 29 580-599 132 132 132 230 230 230 F F F
168242023-12-10T18:56:26.733ZMax gen: 136, Max flush: 235
168252023-12-10T18:56:26.734ZError: Difference in extent metadata found!
168262023-12-10T18:56:26.734ZA Difference in extent metadata is expected here
168272023-12-10T18:56:26.734ZOn loop 68
168282023-12-10T18:56:26.734Z
168292023-12-10T18:56:26.735ZVerifying data now
168302023-12-10T18:56:32.290ZNo change after verify
168312023-12-10T18:56:32.290ZLoop: 68 Downstairs dump after verify (and repair):
168322023-12-10T18:56:32.310ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
168332023-12-10T18:56:32.310Z 0 000-019 136 136 136 235 235 235 F F F
168342023-12-10T18:56:32.314Z 1 020-039 133 133 133 231 231 231 F F F
168352023-12-10T18:56:32.314Z 2 040-059 132 132 132 228 228 228 F F F
168362023-12-10T18:56:32.314Z 3 060-079 132 132 132 228 228 228 F F F
168372023-12-10T18:56:32.314Z 4 080-099 137 137 137 236 236 236 F F F
168382023-12-10T18:56:32.314Z 5 100-119 130 130 130 226 226 226 F F F
168392023-12-10T18:56:32.315Z 6 120-139 136 136 136 235 235 235 F F F
168402023-12-10T18:56:32.315Z 7 140-159 136 136 136 235 235 235 F F F
168412023-12-10T18:56:32.315Z 8 160-179 133 133 133 231 231 231 F F F
168422023-12-10T18:56:32.315Z 9 180-199 137 137 137 236 236 236 F F F
168432023-12-10T18:56:32.315Z 10 200-219 136 136 136 235 235 235 F F F
168442023-12-10T18:56:32.315Z 11 220-239 136 136 136 235 235 235 F F F
168452023-12-10T18:56:32.315Z 12 240-259 137 137 137 236 236 236 F F F
168462023-12-10T18:56:32.315Z 13 260-279 136 136 136 235 235 235 F F F
168472023-12-10T18:56:32.315Z 14 280-299 133 133 133 231 231 231 F F F
168482023-12-10T18:56:32.315Z 15 300-319 135 135 135 234 234 234 F F F
168492023-12-10T18:56:32.315Z 16 320-339 137 137 137 236 236 236 F F F
168502023-12-10T18:56:32.315Z 17 340-359 136 136 136 235 235 235 F F F
168512023-12-10T18:56:32.315Z 18 360-379 137 137 137 236 236 236 F F F
168522023-12-10T18:56:32.315Z 19 380-399 137 137 137 236 236 236 F F F
168532023-12-10T18:56:32.316Z 20 400-419 134 134 134 232 232 232 F F F
168542023-12-10T18:56:32.316Z 21 420-439 136 136 136 235 235 235 F F F
168552023-12-10T18:56:32.316Z 22 440-459 136 136 136 235 235 235 F F F
168562023-12-10T18:56:32.316Z 23 460-479 134 134 134 233 233 233 F F F
168572023-12-10T18:56:32.316Z 24 480-499 137 137 137 236 236 236 F F F
168582023-12-10T18:56:32.316Z 25 500-519 130 130 130 226 226 226 F F F
168592023-12-10T18:56:32.316Z 26 520-539 137 137 137 236 236 236 F F F
168602023-12-10T18:56:32.316Z 27 540-559 130 130 130 226 226 226 F F F
168612023-12-10T18:56:32.316Z 28 560-579 126 126 126 220 220 220 F F F
168622023-12-10T18:56:32.316Z 29 580-599 132 132 132 230 230 230 F F F
168632023-12-10T18:56:32.316ZMax gen: 137, Max flush: 236
168642023-12-10T18:56:32.316Z
168652023-12-10T18:56:32.316ZBegin loop 69
168662023-12-10T18:56:32.318ZDownstairs to restart: 1
168672023-12-10T18:56:32.344Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.350620979Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714}
168682023-12-10T18:56:32.344Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.351044453Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714}
168692023-12-10T18:56:32.349Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.351095704Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714}
168702023-12-10T18:56:32.350Z{"msg":"Crucible 1877d616-3b08-4727-84fd-f010f4eb499c has session id: 90d1bb87-f508-4c30-8d48-d21f050d9fdd","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.351148431Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
168712023-12-10T18:56:32.350Z{"msg":"Upstairs opts: Upstairs UUID: 1877d616-3b08-4727-84fd-f010f4eb499c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.351207362Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
168722023-12-10T18:56:32.350Z{"msg":"Crucible stats registered with UUID: 1877d616-3b08-4727-84fd-f010f4eb499c"Crucible runtime is spawned
168732023-12-10T18:56:32.350Z,"v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.351282102Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
168742023-12-10T18:56:32.350ZThe guest has requested activation with gen:138
168752023-12-10T18:56:32.350Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.351447594Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd","task":"up_listen"}
168762023-12-10T18:56:32.350Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.351521004Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"looper":"0","session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
168772023-12-10T18:56:32.350Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.351595942Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
168782023-12-10T18:56:32.351Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.351626743Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
168792023-12-10T18:56:32.351Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.351778879Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"looper":"2","session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
168802023-12-10T18:56:32.351Z{"msg":"[0] 1877d616-3b08-4727-84fd-f010f4eb499c looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.351852689Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"looper":"0","session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
168812023-12-10T18:56:32.351Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.351951455Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"looper":"1","session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
168822023-12-10T18:56:32.351Z{"msg":"[2] 1877d616-3b08-4727-84fd-f010f4eb499c looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.352025902Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"looper":"2","session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
168832023-12-10T18:56:32.351Z{"msg":"[1] 1877d616-3b08-4727-84fd-f010f4eb499c looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.352058957Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"looper":"1","session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
168842023-12-10T18:56:32.351Z{"msg":"1877d616-3b08-4727-84fd-f010f4eb499c active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.352528368Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
168852023-12-10T18:56:32.351Z{"msg":"Set desired generation to :138","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.352607319Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
168862023-12-10T18:56:32.351Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.352762302Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
168872023-12-10T18:56:32.351Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.353170926Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
168882023-12-10T18:56:32.352Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.353237053Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
168892023-12-10T18:56:32.353Z{"msg":"[0] 1877d616-3b08-4727-84fd-f010f4eb499c (90d1bb87-f508-4c30-8d48-d21f050d9fdd) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.35344391Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
168902023-12-10T18:56:32.353Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.353572942Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
168912023-12-10T18:56:32.353Z{"msg":"[0] client is_active_req TRUE, promote! session 90d1bb87-f508-4c30-8d48-d21f050d9fdd","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.35367026Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
168922023-12-10T18:56:32.353Z{"msg":"[2] 1877d616-3b08-4727-84fd-f010f4eb499c (90d1bb87-f508-4c30-8d48-d21f050d9fdd) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.353763627Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
168932023-12-10T18:56:32.353Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.353791565Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
168942023-12-10T18:56:32.353Z{"msg":"[2] client is_active_req TRUE, promote! session 90d1bb87-f508-4c30-8d48-d21f050d9fdd","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.353813663Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
168952023-12-10T18:56:32.353Z{"msg":"[1] 1877d616-3b08-4727-84fd-f010f4eb499c (90d1bb87-f508-4c30-8d48-d21f050d9fdd) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.353990583Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
168962023-12-10T18:56:32.353Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.354109263Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
168972023-12-10T18:56:32.353Z{"msg":"[1] client is_active_req TRUE, promote! session 90d1bb87-f508-4c30-8d48-d21f050d9fdd","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.354237207Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
168982023-12-10T18:56:32.353Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.354401688Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
168992023-12-10T18:56:32.354Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.354523725Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169002023-12-10T18:56:32.361Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.354565096Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169012023-12-10T18:56:32.361Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.354778999Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169022023-12-10T18:56:32.361Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.354854758Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169032023-12-10T18:56:32.361Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.354905569Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169042023-12-10T18:56:32.361Z{"msg":"1877d616-3b08-4727-84fd-f010f4eb499c WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.354981931Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169052023-12-10T18:56:32.361Z{"msg":"1877d616-3b08-4727-84fd-f010f4eb499c WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.355023737Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169062023-12-10T18:56:32.361Z{"msg":"1877d616-3b08-4727-84fd-f010f4eb499c WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.355091888Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169072023-12-10T18:56:32.361Z{"msg":"[2] 1877d616-3b08-4727-84fd-f010f4eb499c (90d1bb87-f508-4c30-8d48-d21f050d9fdd) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.355388049Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169082023-12-10T18:56:32.363Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.355453023Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169092023-12-10T18:56:32.363Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:32.355517677Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169102023-12-10T18:56:32.363Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.355562507Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169112023-12-10T18:56:32.363Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.355586022Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169122023-12-10T18:56:32.363Z{"msg":"1877d616-3b08-4727-84fd-f010f4eb499c WaitActive WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.355607462Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169132023-12-10T18:56:32.363Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.35565554Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169142023-12-10T18:56:32.363Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.355691675Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169152023-12-10T18:56:32.363Z{"msg":"[1] 1877d616-3b08-4727-84fd-f010f4eb499c (90d1bb87-f508-4c30-8d48-d21f050d9fdd) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.355834715Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169162023-12-10T18:56:32.363Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.355902634Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169172023-12-10T18:56:32.363Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:32.355943791Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169182023-12-10T18:56:32.365Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.355997957Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169192023-12-10T18:56:32.365Z{"msg":"[0] 1877d616-3b08-4727-84fd-f010f4eb499c (90d1bb87-f508-4c30-8d48-d21f050d9fdd) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.356041277Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169202023-12-10T18:56:32.365Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:138
169212023-12-10T18:56:32.365Z,"time":"2023-12-10T18:56:32.356100772Z"Wait for a query_work_queue command to finish before sending IO
169222023-12-10T18:56:32.365Z,"hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169232023-12-10T18:56:32.365Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:32.35614142Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169242023-12-10T18:56:32.365Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.356177696Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169252023-12-10T18:56:32.365Z{"msg":"1877d616-3b08-4727-84fd-f010f4eb499c WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.356256103Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169262023-12-10T18:56:32.365Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.356329299Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169272023-12-10T18:56:32.366Z{"msg":"[0]R flush_numbers[0..12]: [235, 231, 228, 228, 236, 226, 235, 235, 231, 236, 235, 235]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.356370307Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169282023-12-10T18:56:32.366Z{"msg":"[0]R generation[0..12]: [136, 133, 132, 132, 137, 130, 136, 136, 133, 137, 136, 136]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.356450389Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169292023-12-10T18:56:32.366Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.356521502Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169302023-12-10T18:56:32.366Z{"msg":"[1]R flush_numbers[0..12]: [235, 231, 228, 228, 236, 226, 235, 235, 231, 236, 235, 235]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.356549962Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169312023-12-10T18:56:32.366Z{"msg":"[1]R generation[0..12]: [136, 133, 132, 132, 137, 130, 136, 136, 133, 137, 136, 136]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.356600308Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169322023-12-10T18:56:32.366Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.356625107Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169332023-12-10T18:56:32.366Z{"msg":"[2]R flush_numbers[0..12]: [235, 231, 228, 228, 236, 226, 235, 235, 231, 236, 235, 235]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.35664735Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169342023-12-10T18:56:32.366Z{"msg":"[2]R generation[0..12]: [136, 133, 132, 132, 137, 130, 136, 136, 133, 137, 136, 136]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.356701773Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169352023-12-10T18:56:32.366Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.356725919Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169362023-12-10T18:56:32.366Z{"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.356765517Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169372023-12-10T18:56:32.366Z{"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.356787702Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169382023-12-10T18:56:32.366Z{"msg":"Next flush: 237","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
169392023-12-10T18:56:32.366Z,"time":"2023-12-10T18:56:32.356807242Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169402023-12-10T18:56:32.366Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.356899926Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169412023-12-10T18:56:32.366Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.356946778Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169422023-12-10T18:56:32.367Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.356970243Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169432023-12-10T18:56:32.367Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.357010711Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169442023-12-10T18:56:32.367Z{"msg":"1877d616-3b08-4727-84fd-f010f4eb499c is now active with session: 90d1bb87-f508-4c30-8d48-d21f050d9fdd","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.357034719Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169452023-12-10T18:56:32.367Z{"msg":"1877d616-3b08-4727-84fd-f010f4eb499c Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.357075744Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169462023-12-10T18:56:32.367Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.357098363Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169472023-12-10T18:56:32.367Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.357141976Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169482023-12-10T18:56:32.367Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.357173993Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169492023-12-10T18:56:32.367Z{"msg":"1877d616-3b08-4727-84fd-f010f4eb499c Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.3572173Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169502023-12-10T18:56:32.367Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.357239503Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169512023-12-10T18:56:32.367Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.357279299Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169522023-12-10T18:56:32.367Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.357301023Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169532023-12-10T18:56:32.367Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.357321188Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169542023-12-10T18:56:32.367Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.357342994Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169552023-12-10T18:56:32.367Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.357362943Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169562023-12-10T18:56:32.367Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30Loading write count information from file "/tmp/test_repair_verify.data"
169572023-12-10T18:56:32.368Z,"time":"2023-12-10T18:56:32.357420247Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","Run Repair workload
169582023-12-10T18:56:32.368Zpid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169592023-12-10T18:56:32.368Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.357586277Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169602023-12-10T18:56:32.368Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.357613246Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169612023-12-10T18:56:32.368Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:32.357654698Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1714,"session_id":"90d1bb87-f508-4c30-8d48-d21f050d9fdd"}
169622023-12-10T18:56:32.369Z01/30 Write block 95 len 4096 data: 13
169632023-12-10T18:56:32.369Z02/30 Write block 371 len 8192 data: 9 10
169642023-12-10T18:56:32.369Z03/30 Read block 376 len 40960
169652023-12-10T18:56:32.369Z04/30 Write block 491 len 28672 data: 12 7 5 6 6 7 7
169662023-12-10T18:56:32.373Z05/30 Read block 391 len 28672
169672023-12-10T18:56:32.373Z06/30 Read block 66 len 40960
169682023-12-10T18:56:32.373Z07/30 Read block 474 len 36864
169692023-12-10T18:56:32.373Z08/30 Read block 141 len 32768
169702023-12-10T18:56:32.373Z09/30 Read block 6 len 20480
169712023-12-10T18:56:32.373Z10/30 Write block 399 len 36864 data: 9 11 12 12 13 11 13 12 12
169722023-12-10T18:56:32.373Z11/30 Flush
169732023-12-10T18:56:32.376Z12/30 Write block 422 len 20480 data: 12 12 14 11 11
169742023-12-10T18:56:32.379Z13/30 Write block 264 len 24576 data: 11 12 12 13 15 14
169752023-12-10T18:56:32.380Z14/30 Write block 528 len 36864 data: 9 11 10 13 13 13 15 15 10
169762023-12-10T18:56:32.380Z15/30 Write block 313 len 4096 data: 5
169772023-12-10T18:56:32.386Z16/30 Read block 103 len 36864
169782023-12-10T18:56:32.389Z17/30 Read block 58 len 28672
169792023-12-10T18:56:32.389Z18/30 Write block 146 len 12288 data: 8 7 10
169802023-12-10T18:56:32.389Z19/30 Flush
169812023-12-10T18:56:32.392Z20/30 Write block 343 len 28672 data: 13 12 11 11 14 14 17
169822023-12-10T18:56:32.392Z21/30 Read block 307 len 36864
169832023-12-10T18:56:32.395Z22/30 Read block 282 len 32768
169842023-12-10T18:56:32.398Z23/30 Read block 177 len 8192
169852023-12-10T18:56:32.402Z24/30 Read block 179 len 16384
169862023-12-10T18:56:32.402Z25/30 Read block 58 len 8192
169872023-12-10T18:56:32.405Z26/30 Write block 292 len 28672 data: 11 11 8 6 8 8 9
169882023-12-10T18:56:32.405Z27/30 Write block 312 len 20480 data: 5 6 8 8 6
169892023-12-10T18:56:32.405Z28/30 Read block 306 len 4096
169902023-12-10T18:56:32.408Z29/30 Write block 304 len 40960 data: 17 11 12 9 7 7 5 5 6 7
169912023-12-10T18:56:32.408Z30/30 Write block 295 len 16384 data: 7 9 9 10
169922023-12-10T18:56:32.409Z----------------------------------------------------------------
169932023-12-10T18:56:32.409Z Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:30
169942023-12-10T18:56:32.409ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
169952023-12-10T18:56:32.409Z 1 Acked 1000 Write 1 Done Sent Done false
169962023-12-10T18:56:32.409Z 2 Acked 1001 Write 2 Done Sent Done false
169972023-12-10T18:56:32.409Z 3 Acked 1002 Read 10 Done Sent Done false
169982023-12-10T18:56:32.409Z 4 Acked 1003 Write 7 Done Sent Done false
169992023-12-10T18:56:32.409Z 5 Acked 1004 Read 7 Done Sent Done false
170002023-12-10T18:56:32.409Z 6 Acked 1005 Read 10 Done Sent Done false
170012023-12-10T18:56:32.409Z 7 Acked 1006 Read 9 Done Sent Done false
170022023-12-10T18:56:32.409Z 8 Acked 1007 Read 8 Done Sent Done false
170032023-12-10T18:56:32.409Z 9 Acked 1008 Read 5 Done Sent Done false
170042023-12-10T18:56:32.409Z 10 Acked 1009 Write 9 Done Sent Done false
170052023-12-10T18:56:32.409Z 11 Acked 1010 Flush 0 Done Sent Done false
170062023-12-10T18:56:32.409Z 12 Acked 1011 Write 5 Done Sent Done false
170072023-12-10T18:56:32.409Z 13 Acked 1012 Write 6 Done Sent Done false
170082023-12-10T18:56:32.409Z 14 Acked 1013 Write 9 Done Sent Done false
170092023-12-10T18:56:32.410Z 15 Acked 1014 Write 1 Done Sent Done false
170102023-12-10T18:56:32.410Z 16 Acked 1015 Read 9 Done Sent Done false
170112023-12-10T18:56:32.410Z 17 Acked 1016 Read 7 Done Sent Done false
170122023-12-10T18:56:32.410Z 18 Acked 1017 Write 3 Done Sent Done false
170132023-12-10T18:56:32.410Z 19 Acked 1018 Flush 0 Done Sent Done false
170142023-12-10T18:56:32.411Z 20 Acked 1019 Write 7 Done Sent Done false
170152023-12-10T18:56:32.411Z 21 Acked 1020 Read 9 Done Sent Done false
170162023-12-10T18:56:32.411Z 22 Acked 1021 Read 8 Done Sent Done false
170172023-12-10T18:56:32.411Z 23 Acked 1022 Read 2 Done Sent Done false
170182023-12-10T18:56:32.411Z 24 Acked 1023 Read 4 Done Sent Done false
170192023-12-10T18:56:32.411Z 25 Acked 1024 Read 2 Done Sent Done false
170202023-12-10T18:56:32.411Z 26 Acked 1025 Write 7 Done Sent Done false
170212023-12-10T18:56:32.411Z 27 Acked 1026 Write 5 Done Sent Done false
170222023-12-10T18:56:32.411Z 28 Acked 1027 Read 1 Done Sent Done false
170232023-12-10T18:56:32.411Z 29 Acked 1028 Write 10 Sent Sent Sent false
170242023-12-10T18:56:32.411Z 30 Acked 1029 Write 4 Sent Sent Sent false
170252023-12-10T18:56:32.411Z STATES DS:0 DS:1 DS:2 TOTAL
170262023-12-10T18:56:32.411Z New 0 0 0 0
170272023-12-10T18:56:32.411Z Sent 2 30 2 34
170282023-12-10T18:56:32.411Z Done 28 0 28 56
170292023-12-10T18:56:32.411Z Skipped 0 0 0 0
170302023-12-10T18:56:32.411Z Error 0 0 0 0
170312023-12-10T18:56:32.411ZLast Flush: 1018 0 1018
170322023-12-10T18:56:32.412ZDownstairs last five completed:
170332023-12-10T18:56:32.412ZUpstairs last five completed: 30 29 28 27 26 25
170342023-12-10T18:56:32.417ZWrote out file "/tmp/test_repair_verify.data"
170352023-12-10T18:56:32.417Z
170362023-12-10T18:56:32.420ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
170372023-12-10T18:56:32.466ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
170382023-12-10T18:56:32.466Z 0 000-019 136 136 136 235 235 235 F F F
170392023-12-10T18:56:32.466Z 1 020-039 133 133 133 231 231 231 F F F
170402023-12-10T18:56:32.466Z 2 040-059 132 132 132 228 228 228 F F F
170412023-12-10T18:56:32.466Z 3 060-079 132 132 132 228 228 228 F F F
170422023-12-10T18:56:32.466Z 4 080-099 138 137 138 237 236 237 F F F <---
170432023-12-10T18:56:32.470Z 5 100-119 130 130 130 226 226 226 F F F
170442023-12-10T18:56:32.470Z 6 120-139 136 136 136 235 235 235 F F F
170452023-12-10T18:56:32.470Z 7 140-159 138 136 138 238 235 238 F F F <---
170462023-12-10T18:56:32.470Z 8 160-179 133 133 133 231 231 231 F F F
170472023-12-10T18:56:32.470Z 9 180-199 137 137 137 236 236 236 F F F
170482023-12-10T18:56:32.470Z 10 200-219 136 136 136 235 235 235 F F F
170492023-12-10T18:56:32.470Z 11 220-239 136 136 136 235 235 235 F F F
170502023-12-10T18:56:32.470Z 12 240-259 137 137 137 236 236 236 F F F
170512023-12-10T18:56:32.471Z 13 260-279 138 136 138 238 235 238 F F F <---
170522023-12-10T18:56:32.471Z 14 280-299 133 133 133 231 231 231 T F T <---
170532023-12-10T18:56:32.471Z 15 300-319 138 135 138 238 234 238 T F T <---
170542023-12-10T18:56:32.471Z 16 320-339 137 137 137 236 236 236 F F F
170552023-12-10T18:56:32.471Z 17 340-359 136 136 136 235 235 235 T F T <---
170562023-12-10T18:56:32.471Z 18 360-379 138 137 138 237 236 237 F F F <---
170572023-12-10T18:56:32.471Z 19 380-399 138 137 138 237 236 237 F F F <---
170582023-12-10T18:56:32.471Z 20 400-419 138 134 138 237 232 237 F F F <---
170592023-12-10T18:56:32.471Z 21 420-439 138 136 138 238 235 238 F F F <---
170602023-12-10T18:56:32.471Z 22 440-459 136 136 136 235 235 235 F F F
170612023-12-10T18:56:32.471Z 23 460-479 134 134 134 233 233 233 F F F
170622023-12-10T18:56:32.471Z 24 480-499 138 137 138 237 236 237 F F F <---
170632023-12-10T18:56:32.471Z 25 500-519 130 130 130 226 226 226 F F F
170642023-12-10T18:56:32.471Z 26 520-539 138 137 138 238 236 238 F F F <---
170652023-12-10T18:56:32.471Z 27 540-559 130 130 130 226 226 226 F F F
170662023-12-10T18:56:32.471Z 28 560-579 126 126 126 220 220 220 F F F
170672023-12-10T18:56:32.472Z 29 580-599 132 132 132 230 230 230 F F F
170682023-12-10T18:56:32.472ZMax gen: 138, Max flush: 238
170692023-12-10T18:56:32.472ZError: Difference in extent metadata found!
170702023-12-10T18:56:32.472ZA Difference in extent metadata is expected here
170712023-12-10T18:56:32.472ZOn loop 69
170722023-12-10T18:56:32.472Z
170732023-12-10T18:56:32.472ZVerifying data now
170742023-12-10T18:56:37.694ZNo change after verify
170752023-12-10T18:56:37.694ZLoop: 69 Downstairs dump after verify (and repair):
170762023-12-10T18:56:37.703ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
170772023-12-10T18:56:37.703Z 0 000-019 136 136 136 235 235 235 F F F
170782023-12-10T18:56:37.703Z 1 020-039 133 133 133 231 231 231 F F F
170792023-12-10T18:56:37.703Z 2 040-059 132 132 132 228 228 228 F F F
170802023-12-10T18:56:37.703Z 3 060-079 132 132 132 228 228 228 F F F
170812023-12-10T18:56:37.703Z 4 080-099 138 138 138 237 237 237 F F F
170822023-12-10T18:56:37.703Z 5 100-119 130 130 130 226 226 226 F F F
170832023-12-10T18:56:37.703Z 6 120-139 136 136 136 235 235 235 F F F
170842023-12-10T18:56:37.704Z 7 140-159 138 138 138 238 238 238 F F F
170852023-12-10T18:56:37.704Z 8 160-179 133 133 133 231 231 231 F F F
170862023-12-10T18:56:37.704Z 9 180-199 137 137 137 236 236 236 F F F
170872023-12-10T18:56:37.704Z 10 200-219 136 136 136 235 235 235 F F F
170882023-12-10T18:56:37.704Z 11 220-239 136 136 136 235 235 235 F F F
170892023-12-10T18:56:37.704Z 12 240-259 137 137 137 236 236 236 F F F
170902023-12-10T18:56:37.704Z 13 260-279 138 138 138 238 238 238 F F F
170912023-12-10T18:56:37.704Z 14 280-299 139 139 139 239 239 239 F F F
170922023-12-10T18:56:37.704Z 15 300-319 139 139 139 239 239 239 F F F
170932023-12-10T18:56:37.704Z 16 320-339 137 137 137 236 236 236 F F F
170942023-12-10T18:56:37.704Z 17 340-359 139 139 139 239 239 239 F F F
170952023-12-10T18:56:37.704Z 18 360-379 138 138 138 237 237 237 F F F
170962023-12-10T18:56:37.704Z 19 380-399 138 138 138 237 237 237 F F F
170972023-12-10T18:56:37.704Z 20 400-419 138 138 138 237 237 237 F F F
170982023-12-10T18:56:37.704Z 21 420-439 138 138 138 238 238 238 F F F
170992023-12-10T18:56:37.704Z 22 440-459 136 136 136 235 235 235 F F F
171002023-12-10T18:56:37.704Z 23 460-479 134 134 134 233 233 233 F F F
171012023-12-10T18:56:37.704Z 24 480-499 138 138 138 237 237 237 F F F
171022023-12-10T18:56:37.705Z 25 500-519 130 130 130 226 226 226 F F F
171032023-12-10T18:56:37.705Z 26 520-539 138 138 138 238 238 238 F F F
171042023-12-10T18:56:37.706Z 27 540-559 130 130 130 226 226 226 F F F
171052023-12-10T18:56:37.706Z 28 560-579 126 126 126 220 220 220 F F F
171062023-12-10T18:56:37.706Z 29 580-599 132 132 132 230 230 230 F F F
171072023-12-10T18:56:37.706ZMax gen: 139, Max flush: 239
171082023-12-10T18:56:37.706Z
171092023-12-10T18:56:37.706ZBegin loop 70
171102023-12-10T18:56:37.706ZDownstairs to restart: 2
171112023-12-10T18:56:37.706Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.691506301Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722}
171122023-12-10T18:56:37.706Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.691916314Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722}
171132023-12-10T18:56:37.706Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.691983675Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722}
171142023-12-10T18:56:37.706Z{"msg":"Crucible d35b1d8e-952a-432f-9998-00030ecc46d7 has session id: 2fa077ad-9fa5-459d-9a59-03b1b813138a","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.692025313Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"Crucible runtime is spawned
171152023-12-10T18:56:37.708Z}
171162023-12-10T18:56:37.708Z{"msg":"Upstairs opts: Upstairs UUID: d35b1d8e-952a-432f-9998-00030ecc46d7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.692088722Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171172023-12-10T18:56:37.708Z{"msg":"Crucible stats registered with UUID: d35b1d8e-952a-432f-9998-00030ecc46d7","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.692199281Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722The guest has requested activation with gen:140
171182023-12-10T18:56:37.708Z,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171192023-12-10T18:56:37.708Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.692358355Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"looper":"0","session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171202023-12-10T18:56:37.708Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.692678841Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"looper":"2","session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171212023-12-10T18:56:37.708Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.692774332Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a","task":"up_listen"}
171222023-12-10T18:56:37.708Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.692820722Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171232023-12-10T18:56:37.708Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.692890503Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171242023-12-10T18:56:37.708Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.692929382Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"looper":"1","session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171252023-12-10T18:56:37.708Z{"msg":"d35b1d8e-952a-432f-9998-00030ecc46d7 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.693007181Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171262023-12-10T18:56:37.709Z{"msg":"Set desired generation to :140","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.693046026Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171272023-12-10T18:56:37.709Z{"msg":"[2] d35b1d8e-952a-432f-9998-00030ecc46d7 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.693112541Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"looper":"2","session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171282023-12-10T18:56:37.709Z{"msg":"[0] d35b1d8e-952a-432f-9998-00030ecc46d7 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.693187828Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"looper":"0","session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171292023-12-10T18:56:37.709Z{"msg":"[1] d35b1d8e-952a-432f-9998-00030ecc46d7 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.693260519Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"looper":"1","session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171302023-12-10T18:56:37.709Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.693303014Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171312023-12-10T18:56:37.709Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.693369254Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171322023-12-10T18:56:37.709Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.693576525Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171332023-12-10T18:56:37.709Z{"msg":"[0] d35b1d8e-952a-432f-9998-00030ecc46d7 (2fa077ad-9fa5-459d-9a59-03b1b813138a) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.693919449Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171342023-12-10T18:56:37.709Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.694001188Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171352023-12-10T18:56:37.709Z{"msg":"[0] client is_active_req TRUE, promote! session 2fa077ad-9fa5-459d-9a59-03b1b813138a","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.694102268Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171362023-12-10T18:56:37.710Z{"msg":"[1] d35b1d8e-952a-432f-9998-00030ecc46d7 (2fa077ad-9fa5-459d-9a59-03b1b813138a) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.694179028Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171372023-12-10T18:56:37.711Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.694218528Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171382023-12-10T18:56:37.711Z{"msg":"[1] client is_active_req TRUE, promote! session 2fa077ad-9fa5-459d-9a59-03b1b813138a","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.694285479Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171392023-12-10T18:56:37.711Z{"msg":"[2] d35b1d8e-952a-432f-9998-00030ecc46d7 (2fa077ad-9fa5-459d-9a59-03b1b813138a) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.69472083Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171402023-12-10T18:56:37.711Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.694761466Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171412023-12-10T18:56:37.711Z{"msg":"[2] client is_active_req TRUE, promote! session 2fa077ad-9fa5-459d-9a59-03b1b813138a","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.694783918Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171422023-12-10T18:56:37.711Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.695106492Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171432023-12-10T18:56:37.711Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.695288362Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171442023-12-10T18:56:37.711Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.695344212Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171452023-12-10T18:56:37.712Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.695408907Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171462023-12-10T18:56:37.712Z{"msg":"d35b1d8e-952a-432f-9998-00030ecc46d7 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.69543467Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171472023-12-10T18:56:37.712Z{"msg":"d35b1d8e-952a-432f-9998-00030ecc46d7 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.695484314Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171482023-12-10T18:56:37.712Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.69577943Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171492023-12-10T18:56:37.712Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.695848127Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171502023-12-10T18:56:37.712Z{"msg":"d35b1d8e-952a-432f-9998-00030ecc46d7 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.695924118Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171512023-12-10T18:56:37.712Z{"msg":"[1] d35b1d8e-952a-432f-9998-00030ecc46d7 (2fa077ad-9fa5-459d-9a59-03b1b813138a) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.695985418Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171522023-12-10T18:56:37.712Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.69601133Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171532023-12-10T18:56:37.712Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:37.696054391Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171542023-12-10T18:56:37.716Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.696077244Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171552023-12-10T18:56:37.716Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.696118638Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171562023-12-10T18:56:37.716Z{"msg":"d35b1d8e-952a-432f-9998-00030ecc46d7 WaitActive WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.696150518Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171572023-12-10T18:56:37.716Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.696222658Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171582023-12-10T18:56:37.716Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.696298171Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171592023-12-10T18:56:37.717Z{"msg":"[0] d35b1d8e-952a-432f-9998-00030ecc46d7 (2fa077ad-9fa5-459d-9a59-03b1b813138a) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.696342033Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171602023-12-10T18:56:37.717Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.696409168Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171612023-12-10T18:56:37.717Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:37.696450438Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171622023-12-10T18:56:37.717Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.69650892Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171632023-12-10T18:56:37.717Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.696534416Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171642023-12-10T18:56:37.717Z{"msg":"d35b1d8e-952a-432f-9998-00030ecc46d7 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.696576528Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171652023-12-10T18:56:37.717Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.696599131Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171662023-12-10T18:56:37.717Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.696639522Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171672023-12-10T18:56:37.717Z{"msg":"[2] d35b1d8e-952a-432f-9998-00030ecc46d7 (2fa077ad-9fa5-459d-9a59-03b1b813138a) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.696664718Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171682023-12-10T18:56:37.717Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.69675978Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171692023-12-10T18:56:37.717Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:37.69678301Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171702023-12-10T18:56:37.717ZThe guest has finished waiting for activation with:140
171712023-12-10T18:56:37.717Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level"Wait for a query_work_queue command to finish before sending IO
171722023-12-10T18:56:37.717Z:30,"time":"2023-12-10T18:56:37.696810452Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171732023-12-10T18:56:37.717Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.696833156Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171742023-12-10T18:56:37.717Z{"msg":"d35b1d8e-952a-432f-9998-00030ecc46d7 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.696875114Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171752023-12-10T18:56:37.717Z{"msg":"[0]R flush_numbers[0..12]: [235, 231, 228, 228, 237, 226, 235, 238, 231, 236, 235, 235]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.696899433Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171762023-12-10T18:56:37.723Z{"msg":"[0]R generation[0..12]: [136, 133, 132, 132, 138, 130, 136, 138, 133, 137, 136, 136]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.696921058Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171772023-12-10T18:56:37.723Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.696944509Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171782023-12-10T18:56:37.723Z{"msg":"[1]R flush_numbers[0..12]: [235, 231, 228, 228, 237, 226, 235, 238, 231, 236, 235, 235]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.696966312Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171792023-12-10T18:56:37.723Z{"msg":"[1]R generation[0..12]: [136, 133, 132, 132, 138, 130, 136, 138, 133, 137, 136, 136]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.696987658Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171802023-12-10T18:56:37.723Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.69700915Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171812023-12-10T18:56:37.723Z{"msg":"[2]R flush_numbers[0..12]: [235, 231, 228, 228, 237, 226, 235, 238, 231, 236, 235, 235]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.697039061Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171822023-12-10T18:56:37.723Z{"msg":"[2]R generation[0..12]: [136, 133, 132, 132, 138, 130, 136, 138, 133, 137, 136, 136]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.697062095Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171832023-12-10T18:56:37.723Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.697083633Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171842023-12-10T18:56:37.723Z{"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.697139222Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171852023-12-10T18:56:37.723Z{"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.697162927Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171862023-12-10T18:56:37.723Z{"msg":"Next flush: 240","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.697189367Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171872023-12-10T18:56:37.725Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.69720964Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171882023-12-10T18:56:37.725Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.697229366Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171892023-12-10T18:56:37.725Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.6972715Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171902023-12-10T18:56:37.725Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.697294452Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171912023-12-10T18:56:37.725Z{"msg":"d35b1d8e-952a-432f-9998-00030ecc46d7 is now active with session: 2fa077ad-9fa5-459d-9a59-03b1b813138a","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.697316154Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171922023-12-10T18:56:37.725Z{"msg":"d35b1d8e-952a-432f-9998-00030ecc46d7 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.697361644Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171932023-12-10T18:56:37.725Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.697384269Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171942023-12-10T18:56:37.725Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.697403849Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171952023-12-10T18:56:37.725Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.697423663Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171962023-12-10T18:56:37.725Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.697443332Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171972023-12-10T18:56:37.725Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.697463113Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171982023-12-10T18:56:37.725Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.697482924Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
171992023-12-10T18:56:37.725Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.697523036Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
172002023-12-10T18:56:37.726Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.69754483Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
172012023-12-10T18:56:37.726Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.697565011Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
172022023-12-10T18:56:37.726Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.697612399Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
172032023-12-10T18:56:37.726Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:37.697634005Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1722,"session_id":"2fa077ad-9fa5-459d-9a59-03b1b813138a"}
172042023-12-10T18:56:37.726ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
172052023-12-10T18:56:37.726ZLoading write count information from file "/tmp/test_repair_verify.data"
172062023-12-10T18:56:37.726ZRun Repair workload
172072023-12-10T18:56:37.726Z01/30 Write block 518 len 40960 data: 8 9 8 8 8 9 10 9 11 10
172082023-12-10T18:56:37.726Z02/30 Read block 334 len 20480
172092023-12-10T18:56:37.726Z03/30 Read block 306 len 24576
172102023-12-10T18:56:37.726Z04/30 Write block 37 len 28672 data: 12 11 10 9 8 8 8
172112023-12-10T18:56:37.726Z05/30 Read block 230 len 12288
172122023-12-10T18:56:37.726Z06/30 Write block 377 len 8192 data: 16 19
172132023-12-10T18:56:37.726Z07/30 Flush
172142023-12-10T18:56:37.726Z08/30 Write block 359 len 28672 data: 11 9 8 7 8 7 6
172152023-12-10T18:56:37.727Z09/30 Write block 555 len 4096 data: 7
172162023-12-10T18:56:37.727Z10/30 Read block 447 len 36864
172172023-12-10T18:56:37.727Z11/30 Read block 544 len 8192
172182023-12-10T18:56:37.727Z12/30 Flush
172192023-12-10T18:56:37.727Z13/30 Write block 68 len 40960 data: 14 14 12 13 10 13 14 13 13 15
172202023-12-10T18:56:37.727Z14/30 Write block 331 len 16384 data: 10 12 12 10
172212023-12-10T18:56:37.727Z15/30 Read block 202 len 24576
172222023-12-10T18:56:37.727Z16/30 Write block 559 len 16384 data: 9 9 11 13
172232023-12-10T18:56:37.727Z17/30 Read block 467 len 40960
172242023-12-10T18:56:37.731Z18/30 Flush
172252023-12-10T18:56:37.731Z19/30 Write block 547 len 40960 data: 10 11 9 8 6 6 6 6 8 7
172262023-12-10T18:56:37.731Z20/30 Read block 107 len 4096
172272023-12-10T18:56:37.731Z21/30 Write block 355 len 32768 data: 11 13 11 11 12 10 9 8
172282023-12-10T18:56:37.731Z22/30 Read block 296 len 40960
172292023-12-10T18:56:37.735Z23/30 Write block 257 len 28672 data: 7 7 4 8 9 12 13
172302023-12-10T18:56:37.738Z24/30 Read block 135 len 20480
172312023-12-10T18:56:37.738Z25/30 Flush
172322023-12-10T18:56:37.741Z26/30 Write block 336 len 16384 data: 9 9 8 9
172332023-12-10T18:56:37.744Z27/30 Write block 208 len 16384 data: 8 7 6 7
172342023-12-10T18:56:37.744Z28/30 Write block 155 len 8192 data: 9 8
172352023-12-10T18:56:37.747Z29/30 Write block 87 len 36864 data: 15 16 14 11 9 10 12 12 14
172362023-12-10T18:56:37.747Z30/30 Read block 82 len 36864
172372023-12-10T18:56:37.751Z----------------------------------------------------------------
172382023-12-10T18:56:37.751Z Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30
172392023-12-10T18:56:37.751ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
172402023-12-10T18:56:37.753Z 1 Acked 1000 Write 10 Done Done Sent false
172412023-12-10T18:56:37.753Z 2 Acked 1001 Read 5 Done Done Sent false
172422023-12-10T18:56:37.753Z 3 Acked 1002 Read 6 Done Done Sent false
172432023-12-10T18:56:37.753Z 4 Acked 1003 Write 7 Done Done Sent false
172442023-12-10T18:56:37.753Z 5 Acked 1004 Read 3 Done Done Sent false
172452023-12-10T18:56:37.753Z 6 Acked 1005 Write 2 Done Done Sent false
172462023-12-10T18:56:37.753Z 7 Acked 1006 Flush 0 Done Done Sent false
172472023-12-10T18:56:37.754Z 8 Acked 1007 Write 7 Done Done Sent false
172482023-12-10T18:56:37.754Z 9 Acked 1008 Write 1 Done Done Sent false
172492023-12-10T18:56:37.754Z 10 Acked 1009 Read 9 Done Done Sent false
172502023-12-10T18:56:37.754Z 11 Acked 1010 Read 2 Done Done Sent false
172512023-12-10T18:56:37.754Z 12 Acked 1011 Flush 0 Done Done Sent false
172522023-12-10T18:56:37.754Z 13 Acked 1012 Write 10 Done Done Sent false
172532023-12-10T18:56:37.754Z 14 Acked 1013 Write 4 Done Done Sent false
172542023-12-10T18:56:37.754Z 15 Acked 1014 Read 6 Done Done Sent false
172552023-12-10T18:56:37.754Z 16 Acked 1015 Write 4 Done Done Sent false
172562023-12-10T18:56:37.754Z 17 Acked 1016 Read 10 Done Done Sent false
172572023-12-10T18:56:37.754Z 18 Acked 1017 Flush 0 Done Done Sent false
172582023-12-10T18:56:37.754Z 19 Acked 1018 Write 10 Done Done Sent false
172592023-12-10T18:56:37.754Z 20 Acked 1019 Read 1 Done Done Sent false
172602023-12-10T18:56:37.754Z 21 Acked 1020 Write 8 Done Done Sent false
172612023-12-10T18:56:37.754Z 22 Acked 1021 Read 10 Done Done Sent false
172622023-12-10T18:56:37.754Z 23 Acked 1022 Write 7 Done Done Sent false
172632023-12-10T18:56:37.754Z 24 Acked 1023 Read 5 Done Done Sent false
172642023-12-10T18:56:37.754Z 25 Acked 1024 Flush 0 Done Done Sent false
172652023-12-10T18:56:37.755Z 26 Acked 1025 Write 4 Done Done Sent false
172662023-12-10T18:56:37.755Z 27 Acked 1026 Write 4 Done Done Sent false
172672023-12-10T18:56:37.755Z 28 Acked 1027 Write 2 Done Done Sent false
172682023-12-10T18:56:37.755Z 29 Acked 1028 Write 9 Done Done Sent false
172692023-12-10T18:56:37.755Z 30 Acked 1029 Read 9 Sent Done Sent false
172702023-12-10T18:56:37.755Z STATES DS:0 DS:1 DS:2 TOTAL
172712023-12-10T18:56:37.755Z New 0 0 0 0
172722023-12-10T18:56:37.755Z Sent 1 0 30 31
172732023-12-10T18:56:37.755Z Done 29 30 0 59
172742023-12-10T18:56:37.755Z Skipped 0 0 0 0
172752023-12-10T18:56:37.755Z Error 0 0 0 0
172762023-12-10T18:56:37.755ZLast Flush: 1024 1024 0
172772023-12-10T18:56:37.755ZDownstairs last five completed:
172782023-12-10T18:56:37.755ZUpstairs last five completed: 30 29 28 27 26 25
172792023-12-10T18:56:37.755ZWrote out file "/tmp/test_repair_verify.data"
172802023-12-10T18:56:37.760Z
172812023-12-10T18:56:37.764ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
172822023-12-10T18:56:37.783ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
172832023-12-10T18:56:37.783Z 0 000-019 136 136 136 235 235 235 F F F
172842023-12-10T18:56:37.783Z 1 020-039 140 140 133 240 240 231 F F F <---
172852023-12-10T18:56:37.783Z 2 040-059 140 140 132 240 240 228 F F F <---
172862023-12-10T18:56:37.784Z 3 060-079 140 140 132 242 242 228 F F F <---
172872023-12-10T18:56:37.784Z 4 080-099 138 138 138 237 237 237 T T F <---
172882023-12-10T18:56:37.787Z 5 100-119 130 130 130 226 226 226 F F F
172892023-12-10T18:56:37.787Z 6 120-139 136 136 136 235 235 235 F F F
172902023-12-10T18:56:37.788Z 7 140-159 138 138 138 238 238 238 T T F <---
172912023-12-10T18:56:37.788Z 8 160-179 133 133 133 231 231 231 F F F
172922023-12-10T18:56:37.789Z 9 180-199 137 137 137 236 236 236 F F F
172932023-12-10T18:56:37.789Z 10 200-219 136 136 136 235 235 235 T T F <---
172942023-12-10T18:56:37.789Z 11 220-239 136 136 136 235 235 235 F F F
172952023-12-10T18:56:37.789Z 12 240-259 140 140 137 243 243 236 F F F <---
172962023-12-10T18:56:37.789Z 13 260-279 140 140 138 243 243 238 F F F <---
172972023-12-10T18:56:37.789Z 14 280-299 139 139 139 239 239 239 F F F
172982023-12-10T18:56:37.789Z 15 300-319 139 139 139 239 239 239 F F F
172992023-12-10T18:56:37.789Z 16 320-339 140 140 137 242 242 236 T T F <---
173002023-12-10T18:56:37.789Z 17 340-359 140 140 139 243 243 239 F F F <---
173012023-12-10T18:56:37.789Z 18 360-379 140 140 138 243 243 237 F F F <---
173022023-12-10T18:56:37.789Z 19 380-399 138 138 138 237 237 237 F F F
173032023-12-10T18:56:37.789Z 20 400-419 138 138 138 237 237 237 F F F
173042023-12-10T18:56:37.789Z 21 420-439 138 138 138 238 238 238 F F F
173052023-12-10T18:56:37.789Z 22 440-459 136 136 136 235 235 235 F F F
173062023-12-10T18:56:37.790Z 23 460-479 134 134 134 233 233 233 F F F
173072023-12-10T18:56:37.790Z 24 480-499 138 138 138 237 237 237 F F F
173082023-12-10T18:56:37.790Z 25 500-519 140 140 130 240 240 226 F F F <---
173092023-12-10T18:56:37.790Z 26 520-539 140 140 138 240 240 238 F F F <---
173102023-12-10T18:56:37.790Z 27 540-559 140 140 130 243 243 226 F F F <---
173112023-12-10T18:56:37.790Z 28 560-579 140 140 126 242 242 220 F F F <---
173122023-12-10T18:56:37.790Z 29 580-599 132 132 132 230 230 230 F F F
173132023-12-10T18:56:37.790ZMax gen: 140, Max flush: 243
173142023-12-10T18:56:37.790ZError: Difference in extent metadata found!
173152023-12-10T18:56:37.793ZA Difference in extent metadata is expected here
173162023-12-10T18:56:37.794ZOn loop 70
173172023-12-10T18:56:37.794Z
173182023-12-10T18:56:37.794ZVerifying data now
173192023-12-10T18:56:43.180ZNo change after verify
173202023-12-10T18:56:43.180ZLoop: 70 Downstairs dump after verify (and repair):
173212023-12-10T18:56:43.202ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
173222023-12-10T18:56:43.202Z 0 000-019 136 136 136 235 235 235 F F F
173232023-12-10T18:56:43.209Z 1 020-039 140 140 140 240 240 240 F F F
173242023-12-10T18:56:43.209Z 2 040-059 140 140 140 240 240 240 F F F
173252023-12-10T18:56:43.209Z 3 060-079 140 140 140 242 242 242 F F F
173262023-12-10T18:56:43.210Z 4 080-099 141 141 141 244 244 244 F F F
173272023-12-10T18:56:43.210Z 5 100-119 130 130 130 226 226 226 F F F
173282023-12-10T18:56:43.210Z 6 120-139 136 136 136 235 235 235 F F F
173292023-12-10T18:56:43.210Z 7 140-159 141 141 141 244 244 244 F F F
173302023-12-10T18:56:43.210Z 8 160-179 133 133 133 231 231 231 F F F
173312023-12-10T18:56:43.211Z 9 180-199 137 137 137 236 236 236 F F F
173322023-12-10T18:56:43.211Z 10 200-219 141 141 141 244 244 244 F F F
173332023-12-10T18:56:43.211Z 11 220-239 136 136 136 235 235 235 F F F
173342023-12-10T18:56:43.211Z 12 240-259 140 140 140 243 243 243 F F F
173352023-12-10T18:56:43.211Z 13 260-279 140 140 140 243 243 243 F F F
173362023-12-10T18:56:43.211Z 14 280-299 139 139 139 239 239 239 F F F
173372023-12-10T18:56:43.211Z 15 300-319 139 139 139 239 239 239 F F F
173382023-12-10T18:56:43.211Z 16 320-339 141 141 141 244 244 244 F F F
173392023-12-10T18:56:43.211Z 17 340-359 140 140 140 243 243 243 F F F
173402023-12-10T18:56:43.211Z 18 360-379 140 140 140 243 243 243 F F F
173412023-12-10T18:56:43.211Z 19 380-399 138 138 138 237 237 237 F F F
173422023-12-10T18:56:43.212Z 20 400-419 138 138 138 237 237 237 F F F
173432023-12-10T18:56:43.212Z 21 420-439 138 138 138 238 238 238 F F F
173442023-12-10T18:56:43.212Z 22 440-459 136 136 136 235 235 235 F F F
173452023-12-10T18:56:43.212Z 23 460-479 134 134 134 233 233 233 F F F
173462023-12-10T18:56:43.212Z 24 480-499 138 138 138 237 237 237 F F F
173472023-12-10T18:56:43.212Z 25 500-519 140 140 140 240 240 240 F F F
173482023-12-10T18:56:43.212Z 26 520-539 140 140 140 240 240 240 F F F
173492023-12-10T18:56:43.212Z 27 540-559 140 140 140 243 243 243 F F F
173502023-12-10T18:56:43.212Z 28 560-579 140 140 140 242 242 242 F F F
173512023-12-10T18:56:43.212Z 29 580-599 132 132 132 230 230 230 F F F
173522023-12-10T18:56:43.212ZMax gen: 141, Max flush: 244
173532023-12-10T18:56:43.215Z
173542023-12-10T18:56:43.215ZBegin loop 71
173552023-12-10T18:56:43.215ZDownstairs to restart: 2
173562023-12-10T18:56:43.237Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.24135362Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730}
173572023-12-10T18:56:43.241Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.241779022Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730}
173582023-12-10T18:56:43.241Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.241848273Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730}
173592023-12-10T18:56:43.241Z{"msg":"Crucible b89ca868-02c3-4366-a553-75df50b13488 has session id: 61fd339c-bffa-45f0-a731-27f3f2363f8e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.241871512Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173602023-12-10T18:56:43.241Z{"msg":"Upstairs opts: Upstairs UUID: b89ca868-02c3-4366-a553-75df50b13488, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.241993872Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173612023-12-10T18:56:43.241Z{"msg":"Crucible stats registered with UUID: b89ca868-02c3-4366-a553-75df50b13488","v":0,"name":"crucible","level":30Crucible runtime is spawned
173622023-12-10T18:56:43.242Z,"time":"2023-12-10T18:56:43.242045582Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173632023-12-10T18:56:43.242ZThe guest has requested activation with gen:142
173642023-12-10T18:56:43.242Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.242251837Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"looper":"0","session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173652023-12-10T18:56:43.242Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.242362114Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"looper":"1","session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173662023-12-10T18:56:43.242Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.242457458Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e","task":"up_listen"}
173672023-12-10T18:56:43.242Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.242610511Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173682023-12-10T18:56:43.242Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.242744056Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"looper":"2","session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173692023-12-10T18:56:43.242Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.242807971Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173702023-12-10T18:56:43.242Z{"msg":"b89ca868-02c3-4366-a553-75df50b13488 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.242850192Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173712023-12-10T18:56:43.242Z{"msg":"Set desired generation to :142","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.242888526Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173722023-12-10T18:56:43.242Z{"msg":"[0] b89ca868-02c3-4366-a553-75df50b13488 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.242960428Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"looper":"0","session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173732023-12-10T18:56:43.242Z{"msg":"[1] b89ca868-02c3-4366-a553-75df50b13488 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.242987213Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"looper":"1","session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173742023-12-10T18:56:43.242Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.243050057Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173752023-12-10T18:56:43.242Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.243279952Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173762023-12-10T18:56:43.242Z{"msg":"[1] b89ca868-02c3-4366-a553-75df50b13488 (61fd339c-bffa-45f0-a731-27f3f2363f8e) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.243692467Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173772023-12-10T18:56:43.242Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.243803897Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173782023-12-10T18:56:43.243Z{"msg":"[1] client is_active_req TRUE, promote! session 61fd339c-bffa-45f0-a731-27f3f2363f8e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.243864652Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173792023-12-10T18:56:43.243Z{"msg":"[0] b89ca868-02c3-4366-a553-75df50b13488 (61fd339c-bffa-45f0-a731-27f3f2363f8e) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.244019904Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173802023-12-10T18:56:43.243Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.24411568Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173812023-12-10T18:56:43.243Z{"msg":"[0] client is_active_req TRUE, promote! session 61fd339c-bffa-45f0-a731-27f3f2363f8e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.244213185Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173822023-12-10T18:56:43.243Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.244475595Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173832023-12-10T18:56:43.243Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.244559629Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173842023-12-10T18:56:43.243Z{"msg":"b89ca868-02c3-4366-a553-75df50b13488 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.244634234Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173852023-12-10T18:56:43.243Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.244761035Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173862023-12-10T18:56:43.243Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.244829218Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173872023-12-10T18:56:43.243Z{"msg":"b89ca868-02c3-4366-a553-75df50b13488 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.244854646Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173882023-12-10T18:56:43.247Z{"msg":"[1] b89ca868-02c3-4366-a553-75df50b13488 (61fd339c-bffa-45f0-a731-27f3f2363f8e) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.245430302Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173892023-12-10T18:56:43.247Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.245527555Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173902023-12-10T18:56:43.247Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:43.24556475Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173912023-12-10T18:56:43.247Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.245587448Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173922023-12-10T18:56:43.248Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.245632314Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173932023-12-10T18:56:43.248Z{"msg":"b89ca868-02c3-4366-a553-75df50b13488 WaitActive WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.245656185Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173942023-12-10T18:56:43.248Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.245696507Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173952023-12-10T18:56:43.248Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.245718707Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173962023-12-10T18:56:43.248Z{"msg":"[0] b89ca868-02c3-4366-a553-75df50b13488 (61fd339c-bffa-45f0-a731-27f3f2363f8e) WaitActive WaitQuorum New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.245840232Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173972023-12-10T18:56:43.248Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.245891843Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173982023-12-10T18:56:43.248Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:43.245915838Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
173992023-12-10T18:56:43.248Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.245935673Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174002023-12-10T18:56:43.248Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.245955954Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174012023-12-10T18:56:43.248Z{"msg":"b89ca868-02c3-4366-a553-75df50b13488 WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.245976582Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174022023-12-10T18:56:43.248Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.245996474Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174032023-12-10T18:56:43.248Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:43.246016045Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174042023-12-10T18:56:44.240Z{"msg":"[2] b89ca868-02c3-4366-a553-75df50b13488 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.244803365Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"looper":"2","session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174052023-12-10T18:56:44.241Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.24488976Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174062023-12-10T18:56:44.247Z{"msg":"[2] b89ca868-02c3-4366-a553-75df50b13488 (61fd339c-bffa-45f0-a731-27f3f2363f8e) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.245965103Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174072023-12-10T18:56:44.247Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.246022622Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174082023-12-10T18:56:44.247Z{"msg":"[2] client is_active_req TRUE, promote! session 61fd339c-bffa-45f0-a731-27f3f2363f8e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.246048581Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174092023-12-10T18:56:44.247Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.246544487Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174102023-12-10T18:56:44.247Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.246679591Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174112023-12-10T18:56:44.247Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.246743493Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174122023-12-10T18:56:44.247Z{"msg":"b89ca868-02c3-4366-a553-75df50b13488 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.246769247Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174132023-12-10T18:56:44.247Z{"msg":"[2] b89ca868-02c3-4366-a553-75df50b13488 (61fd339c-bffa-45f0-a731-27f3f2363f8e) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.24744081Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174142023-12-10T18:56:44.247Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.247492026Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174152023-12-10T18:56:44.247Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:44.247515579Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174162023-12-10T18:56:44.247Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.24753614Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174172023-12-10T18:56:44.247Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.247562206Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174182023-12-10T18:56:44.247Z{"msg":"b89ca868-02c3-4366-a553-75df50b13488 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.247620468Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174192023-12-10T18:56:44.247Z{"msg":"[0]R flush_numbers[0..12]: [235, 240, 240, 242, 244, 226, 235, 244, 231, 236, 244, 235]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.247645335Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174202023-12-10T18:56:44.248Z{"msg":"[0]R generation[0..12]: [136, 140, 140, 140, 141, 130, 136, 141, 133, 137, 141, 136]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.247687344Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174212023-12-10T18:56:44.248Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.247711436Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174222023-12-10T18:56:44.248Z{"msg":"[1]R flush_numbers[0..12]: [235, 240, 240, 242, 244, 226, 235, 244, 231, 236, 244, 235]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.247733025Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174232023-12-10T18:56:44.248Z{"msg":"[1]R generation[0..12]: [136, 140, 140, 140, 141, 130, 136, 141, 133, 137, 141, 136]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.24775425Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174242023-12-10T18:56:44.248Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.247776434Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174252023-12-10T18:56:44.248Z{"msg":"[2]R flush_numbers[0..12]: [235, 240, 240, 242, 244, 226, 235, 244, 231, 236, 244, 235]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.247817913Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174262023-12-10T18:56:44.248Z{"msg":"[2]R generation[0..12]: [136, 140, 140, 140, 141, 130, 136, 141, 133, 137, 141, 136]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.247840952Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174272023-12-10T18:56:44.248Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.247862552Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174282023-12-10T18:56:44.248Z{"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.247889158Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174292023-12-10T18:56:44.248ZThe guest has finished waiting for activation with:142
174302023-12-10T18:56:44.248Z{"msg":"Wait for a query_work_queue command to finish before sending IO
174312023-12-10T18:56:44.248ZGeneration requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.247956241Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174322023-12-10T18:56:44.248Z{"msg":"Next flush: 245","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.247996738Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174332023-12-10T18:56:44.248Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.248028946Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174342023-12-10T18:56:44.248Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.248090666Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174352023-12-10T18:56:44.248Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.248130282Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174362023-12-10T18:56:44.249Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.248221543Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174372023-12-10T18:56:44.249Z{"msg":"b89ca868-02c3-4366-a553-75df50b13488 is now active with session: 61fd339c-bffa-45f0-a731-27f3f2363f8e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.248267544Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174382023-12-10T18:56:44.249Z{"msg":"b89ca868-02c3-4366-a553-75df50b13488 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.248324228Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174392023-12-10T18:56:44.250Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.248362757Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174402023-12-10T18:56:44.250Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.248398668Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174412023-12-10T18:56:44.250Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.24848484Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174422023-12-10T18:56:44.250Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.248526441Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174432023-12-10T18:56:44.250Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.248579432Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174442023-12-10T18:56:44.250Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.248618175Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174452023-12-10T18:56:44.250Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.248682062Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174462023-12-10T18:56:44.250Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.24872014Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174472023-12-10T18:56:44.250Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.248771712Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174482023-12-10T18:56:44.250Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
174492023-12-10T18:56:44.251Z,"time":"2023-12-10T18:56:44.248795265Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174502023-12-10T18:56:44.251Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:44.248895073Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1730,"session_id":"61fd339c-bffa-45f0-a731-27f3f2363f8e"}
174512023-12-10T18:56:44.251ZLoading write count information from file "/tmp/test_repair_verify.data"
174522023-12-10T18:56:44.251ZRun Repair workload
174532023-12-10T18:56:44.255Z01/30 Write block 199 len 28672 data: 14 13 15 14 16 14 13
174542023-12-10T18:56:44.255Z02/30 Write block 232 len 16384 data: 16 13 16 15
174552023-12-10T18:56:44.255Z03/30 Write block 441 len 12288 data: 9 7 9
174562023-12-10T18:56:44.255Z04/30 Read block 252 len 8192
174572023-12-10T18:56:44.255Z05/30 Write block 493 len 24576 data: 6 7 7 8 8 9
174582023-12-10T18:56:44.255Z06/30 Flush
174592023-12-10T18:56:44.259Z07/30 Write block 521 len 36864 data: 9 9 10 11 10 12 11 10 12
174602023-12-10T18:56:44.259Z08/30 Flush
174612023-12-10T18:56:44.262Z09/30 Write block 22 len 16384 data: 9 8 9 8
174622023-12-10T18:56:44.262Z10/30 Write block 235 len 24576 data: 16 14 13 12 15 14
174632023-12-10T18:56:44.262Z11/30 Read block 492 len 12288
174642023-12-10T18:56:44.267Z12/30 Read block 210 len 12288
174652023-12-10T18:56:44.268Z13/30 Flush
174662023-12-10T18:56:44.282Z14/30 Write block 578 len 8192 data: 9 10
174672023-12-10T18:56:44.282Z15/30 Read block 90 len 20480
174682023-12-10T18:56:44.282Z16/30 Read block 24 len 32768
174692023-12-10T18:56:44.282Z17/30 Write block 339 len 40960 data: 10 9 10 11 14 13 12 12 15 15
174702023-12-10T18:56:44.282Z18/30 Read block 113 len 16384
174712023-12-10T18:56:44.285Z19/30 Read block 454 len 40960
174722023-12-10T18:56:44.286Z20/30 Read block 597 len 4096
174732023-12-10T18:56:44.286Z21/30 Read block 263 len 8192
174742023-12-10T18:56:44.286Z22/30 Flush
174752023-12-10T18:56:44.286Z23/30 Write block 247 len 20480 data: 12 13 14 13 13
174762023-12-10T18:56:44.290Z24/30 Read block 111 len 24576
174772023-12-10T18:56:44.290Z25/30 Read block 148 len 40960
174782023-12-10T18:56:44.294Z26/30 Write block 395 len 32768 data: 7 7 7 8 10 12 13 13
174792023-12-10T18:56:44.294Z27/30 Read block 368 len 32768
174802023-12-10T18:56:44.297Z28/30 Write block 17 len 36864 data: 14 10 11 11 9 10 9 10 9
174812023-12-10T18:56:44.300Z29/30 Write block 397 len 28672 data: 8 9 11 13 14 14 14
174822023-12-10T18:56:44.300Z30/30 Write block 344 len 4096 data: 14
174832023-12-10T18:56:44.300Z----------------------------------------------------------------
174842023-12-10T18:56:44.300Z Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:30
174852023-12-10T18:56:44.300ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
174862023-12-10T18:56:44.300Z 1 Acked 1000 Write 7 Done Done Done false
174872023-12-10T18:56:44.300Z 2 Acked 1001 Write 4 Done Done Done false
174882023-12-10T18:56:44.300Z 3 Acked 1002 Write 3 Done Done Done false
174892023-12-10T18:56:44.301Z 4 Acked 1003 Read 2 Done Done Done false
174902023-12-10T18:56:44.301Z 5 Acked 1004 Write 6 Done Done Done false
174912023-12-10T18:56:44.301Z 6 Acked 1005 Flush 0 Done Done Sent false
174922023-12-10T18:56:44.301Z 7 Acked 1006 Write 9 Done Done Sent false
174932023-12-10T18:56:44.301Z 8 Acked 1007 Flush 0 Done Done Sent false
174942023-12-10T18:56:44.301Z 9 Acked 1008 Write 4 Done Done Sent false
174952023-12-10T18:56:44.301Z 10 Acked 1009 Write 6 Done Done Sent false
174962023-12-10T18:56:44.301Z 11 Acked 1010 Read 3 Done Done Sent false
174972023-12-10T18:56:44.301Z 12 Acked 1011 Read 3 Done Done Sent false
174982023-12-10T18:56:44.301Z 13 Acked 1012 Flush 0 Done Done Sent false
174992023-12-10T18:56:44.301Z 14 Acked 1013 Write 2 Done Done Sent false
175002023-12-10T18:56:44.301Z 15 Acked 1014 Read 5 Done Done Sent false
175012023-12-10T18:56:44.301Z 16 Acked 1015 Read 8 Done Done Sent false
175022023-12-10T18:56:44.301Z 17 Acked 1016 Write 10 Done Done Sent false
175032023-12-10T18:56:44.301Z 18 Acked 1017 Read 4 Done Done Sent false
175042023-12-10T18:56:44.301Z 19 Acked 1018 Read 10 Done Done Sent false
175052023-12-10T18:56:44.301Z 20 Acked 1019 Read 1 Done Done Sent false
175062023-12-10T18:56:44.301Z 21 Acked 1020 Read 2 Done Done Sent false
175072023-12-10T18:56:44.302Z 22 Acked 1021 Flush 0 Done Done Sent false
175082023-12-10T18:56:44.302Z 23 Acked 1022 Write 5 Done Done Sent false
175092023-12-10T18:56:44.302Z 24 Acked 1023 Read 6 Done Done Sent false
175102023-12-10T18:56:44.302Z 25 Acked 1024 Read 10 Done Done Sent false
175112023-12-10T18:56:44.302Z 26 Acked 1025 Write 8 Done Done Sent false
175122023-12-10T18:56:44.302Z 27 Acked 1026 Read 8 Done Done Sent false
175132023-12-10T18:56:44.302Z 28 Acked 1027 Write 9 Done Done Sent false
175142023-12-10T18:56:44.302Z 29 Acked 1028 Write 7 Done Sent Sent false
175152023-12-10T18:56:44.302Z 30 Acked 1029 Write 1 Sent Sent Sent false
175162023-12-10T18:56:44.302Z STATES DS:0 DS:1 DS:2 TOTAL
175172023-12-10T18:56:44.302Z New 0 0 0 0
175182023-12-10T18:56:44.302Z Sent 1 2 25 28
175192023-12-10T18:56:44.303Z Done 29 28 5 62
175202023-12-10T18:56:44.303Z Skipped 0 0 0 0
175212023-12-10T18:56:44.303Z Error 0 0 0 0
175222023-12-10T18:56:44.303ZLast Flush: 1021 1021 0
175232023-12-10T18:56:44.303ZDownstairs last five completed:
175242023-12-10T18:56:44.303ZUpstairs last five completed: 30 29 28 27 26 25
175252023-12-10T18:56:44.306ZWrote out file "/tmp/test_repair_verify.data"
175262023-12-10T18:56:44.309Z
175272023-12-10T18:56:44.314ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
175282023-12-10T18:56:44.341ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
175292023-12-10T18:56:44.341Z 0 000-019 136 136 136 235 235 235 T T F <---
175302023-12-10T18:56:44.341Z 1 020-039 142 142 140 247 247 240 T T F <---
175312023-12-10T18:56:44.342Z 2 040-059 140 140 140 240 240 240 F F F
175322023-12-10T18:56:44.342Z 3 060-079 140 140 140 242 242 242 F F F
175332023-12-10T18:56:44.342Z 4 080-099 141 141 141 244 244 244 F F F
175342023-12-10T18:56:44.342Z 5 100-119 130 130 130 226 226 226 F F F
175352023-12-10T18:56:44.342Z 6 120-139 136 136 136 235 235 235 F F F
175362023-12-10T18:56:44.345Z 7 140-159 141 141 141 244 244 244 F F F
175372023-12-10T18:56:44.345Z 8 160-179 133 133 133 231 231 231 F F F
175382023-12-10T18:56:44.345Z 9 180-199 142 142 137 245 245 236 F F T <---
175392023-12-10T18:56:44.345Z 10 200-219 142 142 141 245 245 244 F F T <---
175402023-12-10T18:56:44.345Z 11 220-239 142 142 136 247 247 235 F F T <---
175412023-12-10T18:56:44.345Z 12 240-259 142 142 140 247 247 243 T T F <---
175422023-12-10T18:56:44.345Z 13 260-279 140 140 140 243 243 243 F F F
175432023-12-10T18:56:44.345Z 14 280-299 139 139 139 239 239 239 F F F
175442023-12-10T18:56:44.346Z 15 300-319 139 139 139 239 239 239 F F F
175452023-12-10T18:56:44.346Z 16 320-339 142 142 141 248 248 244 F F F <---
175462023-12-10T18:56:44.346Z 17 340-359 142 142 140 248 248 243 T T F <---
175472023-12-10T18:56:44.346Z 18 360-379 140 140 140 243 243 243 F F F
175482023-12-10T18:56:44.346Z 19 380-399 138 138 138 237 237 237 T T F <---
175492023-12-10T18:56:44.346Z 20 400-419 138 138 138 237 237 237 T T F <---
175502023-12-10T18:56:44.346Z 21 420-439 138 138 138 238 238 238 F F F
175512023-12-10T18:56:44.346Z 22 440-459 142 142 136 245 245 235 F F T <---
175522023-12-10T18:56:44.346Z 23 460-479 134 134 134 233 233 233 F F F
175532023-12-10T18:56:44.346Z 24 480-499 142 142 138 245 245 237 F F T <---
175542023-12-10T18:56:44.346Z 25 500-519 140 140 140 240 240 240 F F F
175552023-12-10T18:56:44.346Z 26 520-539 142 142 140 246 246 240 F F F <---
175562023-12-10T18:56:44.346Z 27 540-559 140 140 140 243 243 243 F F F
175572023-12-10T18:56:44.346Z 28 560-579 142 142 140 248 248 242 F F F <---
175582023-12-10T18:56:44.346Z 29 580-599 132 132 132 230 230 230 F F F
175592023-12-10T18:56:44.346ZMax gen: 142, Max flush: 248
175602023-12-10T18:56:44.346ZError: Difference in extent metadata found!
175612023-12-10T18:56:44.350ZA Difference in extent metadata is expected here
175622023-12-10T18:56:44.350ZOn loop 71
175632023-12-10T18:56:44.350Z
175642023-12-10T18:56:44.350ZVerifying data now
175652023-12-10T18:56:50.687ZNo change after verify
175662023-12-10T18:56:50.688ZLoop: 71 Downstairs dump after verify (and repair):
175672023-12-10T18:56:50.706ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
175682023-12-10T18:56:50.707Z 0 000-019 143 143 143 249 249 249 F F F
175692023-12-10T18:56:50.708Z 1 020-039 143 143 143 249 249 249 F F F
175702023-12-10T18:56:50.708Z 2 040-059 140 140 140 240 240 240 F F F
175712023-12-10T18:56:50.708Z 3 060-079 140 140 140 242 242 242 F F F
175722023-12-10T18:56:50.709Z 4 080-099 141 141 141 244 244 244 F F F
175732023-12-10T18:56:50.709Z 5 100-119 130 130 130 226 226 226 F F F
175742023-12-10T18:56:50.709Z 6 120-139 136 136 136 235 235 235 F F F
175752023-12-10T18:56:50.709Z 7 140-159 141 141 141 244 244 244 F F F
175762023-12-10T18:56:50.714Z 8 160-179 133 133 133 231 231 231 F F F
175772023-12-10T18:56:50.715Z 9 180-199 142 142 142 245 245 245 F F F
175782023-12-10T18:56:50.715Z 10 200-219 142 142 142 245 245 245 F F F
175792023-12-10T18:56:50.715Z 11 220-239 142 142 142 247 247 247 F F F
175802023-12-10T18:56:50.715Z 12 240-259 143 143 143 249 249 249 F F F
175812023-12-10T18:56:50.715Z 13 260-279 140 140 140 243 243 243 F F F
175822023-12-10T18:56:50.715Z 14 280-299 139 139 139 239 239 239 F F F
175832023-12-10T18:56:50.716Z 15 300-319 139 139 139 239 239 239 F F F
175842023-12-10T18:56:50.716Z 16 320-339 142 142 142 248 248 248 F F F
175852023-12-10T18:56:50.716Z 17 340-359 143 143 143 249 249 249 F F F
175862023-12-10T18:56:50.716Z 18 360-379 140 140 140 243 243 243 F F F
175872023-12-10T18:56:50.716Z 19 380-399 143 143 143 249 249 249 F F F
175882023-12-10T18:56:50.716Z 20 400-419 143 143 143 249 249 249 F F F
175892023-12-10T18:56:50.717Z 21 420-439 138 138 138 238 238 238 F F F
175902023-12-10T18:56:50.717Z 22 440-459 142 142 142 245 245 245 F F F
175912023-12-10T18:56:50.717Z 23 460-479 134 134 134 233 233 233 F F F
175922023-12-10T18:56:50.717Z 24 480-499 142 142 142 245 245 245 F F F
175932023-12-10T18:56:50.717Z 25 500-519 140 140 140 240 240 240 F F F
175942023-12-10T18:56:50.717Z 26 520-539 142 142 142 246 246 246 F F F
175952023-12-10T18:56:50.717Z 27 540-559 140 140 140 243 243 243 F F F
175962023-12-10T18:56:50.718Z 28 560-579 142 142 142 248 248 248 F F F
175972023-12-10T18:56:50.718Z 29 580-599 132 132 132 230 230 230 F F F
175982023-12-10T18:56:50.718ZMax gen: 143, Max flush: 249
175992023-12-10T18:56:50.718Z
176002023-12-10T18:56:50.718ZBegin loop 72
176012023-12-10T18:56:50.718ZDownstairs to restart: 1
176022023-12-10T18:56:50.743Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.743988888Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738}
176032023-12-10T18:56:50.743Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.744381916Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738}
176042023-12-10T18:56:50.751Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.74443389Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738}
176052023-12-10T18:56:50.751Z{"msg":"Crucible eecf204e-2b6f-42d5-99a3-39af51dd1fe0 has session id: 1ee3972c-f421-4de5-be28-62738ad4d5cb","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.744499128Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176062023-12-10T18:56:50.751Z{"msg":"Upstairs opts: Upstairs UUID: eecf204e-2b6f-42d5-99a3-39af51dd1fe0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
176072023-12-10T18:56:50.751Z,"time":"2023-12-10T18:56:50.744566003Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176082023-12-10T18:56:50.751Z{"msg":"Crucible stats registered with UUID: eecf204e-2b6f-42d5-99a3-39af51dd1fe0","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.744617998Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176092023-12-10T18:56:50.751ZThe guest has requested activation with gen:144
176102023-12-10T18:56:50.752Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.74481007Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"looper":"0","session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176112023-12-10T18:56:50.752Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.744893385Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb","task":"up_listen"}
176122023-12-10T18:56:50.752Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.744937271Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176132023-12-10T18:56:50.753Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.7450036Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176142023-12-10T18:56:50.753Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.745115416Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"looper":"2","session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176152023-12-10T18:56:50.753Z{"msg":"[0] eecf204e-2b6f-42d5-99a3-39af51dd1fe0 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.745227436Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"looper":"0","session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176162023-12-10T18:56:50.754Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.745335308Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"looper":"1","session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176172023-12-10T18:56:50.754Z{"msg":"[1] eecf204e-2b6f-42d5-99a3-39af51dd1fe0 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.745459118Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"looper":"1","session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176182023-12-10T18:56:50.754Z{"msg":"eecf204e-2b6f-42d5-99a3-39af51dd1fe0 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.745514364Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176192023-12-10T18:56:50.754Z{"msg":"Set desired generation to :144","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.745553646Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176202023-12-10T18:56:50.754Z{"msg":"[2] eecf204e-2b6f-42d5-99a3-39af51dd1fe0 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.745690832Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"looper":"2","session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176212023-12-10T18:56:50.754Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.745752834Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176222023-12-10T18:56:50.754Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.746125247Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176232023-12-10T18:56:50.754Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.746246322Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176242023-12-10T18:56:50.754Z{"msg":"[0] eecf204e-2b6f-42d5-99a3-39af51dd1fe0 (1ee3972c-f421-4de5-be28-62738ad4d5cb) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.746464359Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176252023-12-10T18:56:50.754Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.746527133Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176262023-12-10T18:56:50.754Z{"msg":"[0] client is_active_req TRUE, promote! session 1ee3972c-f421-4de5-be28-62738ad4d5cb","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.746558973Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176272023-12-10T18:56:50.754Z{"msg":"[2] eecf204e-2b6f-42d5-99a3-39af51dd1fe0 (1ee3972c-f421-4de5-be28-62738ad4d5cb) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.746636412Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176282023-12-10T18:56:50.754Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.74666802Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176292023-12-10T18:56:50.754Z{"msg":"[2] client is_active_req TRUE, promote! session 1ee3972c-f421-4de5-be28-62738ad4d5cb","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.746745355Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176302023-12-10T18:56:50.754Z{"msg":"[1] eecf204e-2b6f-42d5-99a3-39af51dd1fe0 (1ee3972c-f421-4de5-be28-62738ad4d5cb) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.747220048Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176312023-12-10T18:56:50.754Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.747346103Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176322023-12-10T18:56:50.755Z{"msg":"[1] client is_active_req TRUE, promote! session 1ee3972c-f421-4de5-be28-62738ad4d5cb","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.747374056Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176332023-12-10T18:56:50.755Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.747401704Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176342023-12-10T18:56:50.755Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.747480283Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176352023-12-10T18:56:50.755Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.747507546Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176362023-12-10T18:56:50.755Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.747570482Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176372023-12-10T18:56:50.755Z{"msg":"eecf204e-2b6f-42d5-99a3-39af51dd1fe0 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.747598144Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176382023-12-10T18:56:50.755Z{"msg":"eecf204e-2b6f-42d5-99a3-39af51dd1fe0 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.747675342Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176392023-12-10T18:56:50.755Z{"msg":"[0] eecf204e-2b6f-42d5-99a3-39af51dd1fe0 (1ee3972c-f421-4de5-be28-62738ad4d5cb) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.748018931Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176402023-12-10T18:56:50.755Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.748088123Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176412023-12-10T18:56:50.755Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:50.748112853Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176422023-12-10T18:56:50.755Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.748133091Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176432023-12-10T18:56:50.755Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.748155939Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176442023-12-10T18:56:50.755Z{"msg":"eecf204e-2b6f-42d5-99a3-39af51dd1fe0 WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.748177693Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176452023-12-10T18:56:50.755Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.748211977Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176462023-12-10T18:56:50.755Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.748260887Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176472023-12-10T18:56:50.755Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.748352799Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176482023-12-10T18:56:50.755Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.748435082Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176492023-12-10T18:56:50.755Z{"msg":"eecf204e-2b6f-42d5-99a3-39af51dd1fe0 WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.748515039Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176502023-12-10T18:56:50.756Z{"msg":"[2] eecf204e-2b6f-42d5-99a3-39af51dd1fe0 (1ee3972c-f421-4de5-be28-62738ad4d5cb) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.748593471Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176512023-12-10T18:56:50.756Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.748664496Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176522023-12-10T18:56:50.756Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:50.748734091Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176532023-12-10T18:56:50.757Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.748803754Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176542023-12-10T18:56:50.757Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.748875706Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176552023-12-10T18:56:50.757Z{"msg":"eecf204e-2b6f-42d5-99a3-39af51dd1fe0 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.748972108Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176562023-12-10T18:56:50.757Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.749011371Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176572023-12-10T18:56:50.757Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.749047432Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176582023-12-10T18:56:50.757Z{"msg":"[1] eecf204e-2b6f-42d5-99a3-39af51dd1fe0 (1ee3972c-f421-4de5-be28-62738ad4d5cb) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.74933603Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176592023-12-10T18:56:50.757Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.749395444Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176602023-12-10T18:56:50.757Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:50.749420269Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176612023-12-10T18:56:50.757Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.749478843Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176622023-12-10T18:56:50.758Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.749509267Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176632023-12-10T18:56:50.758Z{"msg":"eecf204e-2b6f-42d5-99a3-39af51dd1fe0 WaitQuorum WaitQuorum WaitQuorum"The guest has finished waiting for activation with:144
176642023-12-10T18:56:50.758Z,"v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
176652023-12-10T18:56:50.758Z,"time":"2023-12-10T18:56:50.749569299Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176662023-12-10T18:56:50.758Z{"msg":"[0]R flush_numbers[0..12]: [249, 249, 240, 242, 244, 226, 235, 244, 231, 245, 245, 247]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.74959579Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176672023-12-10T18:56:50.758Z{"msg":"[0]R generation[0..12]: [143, 143, 140, 140, 141, 130, 136, 141, 133, 142, 142, 142]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.74962334Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176682023-12-10T18:56:50.758Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.749684013Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176692023-12-10T18:56:50.758Z{"msg":"[1]R flush_numbers[0..12]: [249, 249, 240, 242, 244, 226, 235, 244, 231, 245, 245, 247]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.749709632Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176702023-12-10T18:56:50.759Z{"msg":"[1]R generation[0..12]: [143, 143, 140, 140, 141, 130, 136, 141, 133, 142, 142, 142]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.749743483Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176712023-12-10T18:56:50.759Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.749818112Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176722023-12-10T18:56:50.759Z{"msg":"[2]R flush_numbers[0..12]: [249, 249, 240, 242, 244, 226, 235, 244, 231, 245, 245, 247]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.74984431Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176732023-12-10T18:56:50.759Z{"msg":"[2]R generation[0..12]: [143, 143, 140, 140, 141, 130, 136, 141, 133, 142, 142, 142]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.749900124Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176742023-12-10T18:56:50.759Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.749927356Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176752023-12-10T18:56:50.760Z{"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.749946928Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176762023-12-10T18:56:50.760Z{"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.749969524Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176772023-12-10T18:56:50.760Z{"msg":"Next flush: 250","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.750022805Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176782023-12-10T18:56:50.760Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.750045421Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176792023-12-10T18:56:50.760Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.750065266Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176802023-12-10T18:56:50.760Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.750112477Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176812023-12-10T18:56:50.760Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.750134472Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176822023-12-10T18:56:50.760Z{"msg":"eecf204e-2b6f-42d5-99a3-39af51dd1fe0 is now active with session: 1ee3972c-f421-4de5-be28-62738ad4d5cb","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.750155858Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176832023-12-10T18:56:50.760Z{"msg":"eecf204e-2b6f-42d5-99a3-39af51dd1fe0 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.75019627Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176842023-12-10T18:56:50.760Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.750218714Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176852023-12-10T18:56:50.760Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.750238222Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176862023-12-10T18:56:50.760Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.75028691Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176872023-12-10T18:56:50.760Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.750308946Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176882023-12-10T18:56:50.760Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.750328427Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176892023-12-10T18:56:50.761Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.750373047Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176902023-12-10T18:56:50.761Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.750394542Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176912023-12-10T18:56:50.761Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.75041371Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176922023-12-10T18:56:50.761Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.750433589Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176932023-12-10T18:56:50.764Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.750476225Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176942023-12-10T18:56:50.764Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:50.750522499Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1738,"session_id":"1ee3972c-f421-4de5-be28-62738ad4d5cb"}
176952023-12-10T18:56:50.764ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
176962023-12-10T18:56:50.765ZLoading write count information from file "/tmp/test_repair_verify.data"
176972023-12-10T18:56:50.765ZRun Repair workload
176982023-12-10T18:56:50.765Z01/30 Write block 179 len 4096 data: 6
176992023-12-10T18:56:50.765Z02/30 Read block 326 len 28672
177002023-12-10T18:56:50.765Z03/30 Read block 21 len 4096
177012023-12-10T18:56:50.765Z04/30 Write block 573 len 24576 data: 10 11 12 12 12 10
177022023-12-10T18:56:50.765Z05/30 Flush
177032023-12-10T18:56:50.765Z06/30 Write block 529 len 12288 data: 13 11 14
177042023-12-10T18:56:50.765Z07/30 Read block 250 len 12288
177052023-12-10T18:56:50.765Z08/30 Read block 75 len 8192
177062023-12-10T18:56:50.765Z09/30 Read block 195 len 40960
177072023-12-10T18:56:50.765Z10/30 Write block 326 len 12288 data: 11 12 11
177082023-12-10T18:56:50.768Z11/30 Write block 343 len 12288 data: 15 15 13
177092023-12-10T18:56:50.768Z12/30 Write block 226 len 4096 data: 15
177102023-12-10T18:56:50.768Z13/30 Write block 296 len 32768 data: 10 10 11 13 9 12 12 15
177112023-12-10T18:56:50.768Z14/30 Read block 415 len 28672
177122023-12-10T18:56:50.772Z15/30 Read block 169 len 8192
177132023-12-10T18:56:50.775Z16/30 Read block 97 len 40960
177142023-12-10T18:56:50.778Z17/30 Read block 208 len 32768
177152023-12-10T18:56:50.778Z18/30 Read block 331 len 8192
177162023-12-10T18:56:50.781Z19/30 Read block 256 len 36864
177172023-12-10T18:56:50.784Z20/30 Read block 299 len 32768
177182023-12-10T18:56:50.787Z21/30 Flush
177192023-12-10T18:56:50.791Z22/30 Write block 209 len 32768 data: 8 7 8 5 7 10 12 15
177202023-12-10T18:56:50.791Z23/30 Write block 101 len 4096 data: 10
177212023-12-10T18:56:50.794Z24/30 Write block 390 len 28672 data: 11 9 10 8 7 8 8
177222023-12-10T18:56:50.794Z25/30 Read block 497 len 20480
177232023-12-10T18:56:50.794Z26/30 Read block 316 len 36864
177242023-12-10T18:56:50.797Z27/30 Read block 67 len 16384
177252023-12-10T18:56:50.800Z28/30 Read block 516 len 36864
177262023-12-10T18:56:50.802Z29/30 Read block 104 len 16384
177272023-12-10T18:56:50.806Z30/30 Write block 151 len 24576 data: 9 9 9 14 10 9
177282023-12-10T18:56:50.807Z----------------------------------------------------------------
177292023-12-10T18:56:50.807Z Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30
177302023-12-10T18:56:50.807ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
177312023-12-10T18:56:50.807Z 1 Acked 1000 Write 1 Done Sent Done false
177322023-12-10T18:56:50.807Z 2 Acked 1001 Read 7 Done Sent Done false
177332023-12-10T18:56:50.807Z 3 Acked 1002 Read 1 Done Sent Done false
177342023-12-10T18:56:50.807Z 4 Acked 1003 Write 6 Done Sent Done false
177352023-12-10T18:56:50.807Z 5 Acked 1004 Flush 0 Done Sent Done false
177362023-12-10T18:56:50.807Z 6 Acked 1005 Write 3 Done Sent Done false
177372023-12-10T18:56:50.807Z 7 Acked 1006 Read 3 Done Sent Done false
177382023-12-10T18:56:50.807Z 8 Acked 1007 Read 2 Done Sent Done false
177392023-12-10T18:56:50.807Z 9 Acked 1008 Read 10 Done Sent Done false
177402023-12-10T18:56:50.807Z 10 Acked 1009 Write 3 Done Sent Done false
177412023-12-10T18:56:50.807Z 11 Acked 1010 Write 3 Done Sent Done false
177422023-12-10T18:56:50.807Z 12 Acked 1011 Write 1 Done Sent Done false
177432023-12-10T18:56:50.807Z 13 Acked 1012 Write 8 Done Sent Done false
177442023-12-10T18:56:50.807Z 14 Acked 1013 Read 7 Done Sent Done false
177452023-12-10T18:56:50.808Z 15 Acked 1014 Read 2 Done Sent Done false
177462023-12-10T18:56:50.808Z 16 Acked 1015 Read 10 Done Sent Done false
177472023-12-10T18:56:50.808Z 17 Acked 1016 Read 8 Done Sent Done false
177482023-12-10T18:56:50.808Z 18 Acked 1017 Read 2 Done Sent Done false
177492023-12-10T18:56:50.808Z 19 Acked 1018 Read 9 Done Sent Done false
177502023-12-10T18:56:50.808Z 20 Acked 1019 Read 8 Done Sent Done false
177512023-12-10T18:56:50.809Z 21 Acked 1020 Flush 0 Done Sent Done false
177522023-12-10T18:56:50.809Z 22 Acked 1021 Write 8 Done Sent Done false
177532023-12-10T18:56:50.809Z 23 Acked 1022 Write 1 Done Sent Done false
177542023-12-10T18:56:50.809Z 24 Acked 1023 Write 7 Done Sent Done false
177552023-12-10T18:56:50.809Z 25 Acked 1024 Read 5 Done Sent Done false
177562023-12-10T18:56:50.809Z 26 Acked 1025 Read 9 Done Sent Done false
177572023-12-10T18:56:50.809Z 27 Acked 1026 Read 4 Done Sent Done false
177582023-12-10T18:56:50.809Z 28 Acked 1027 Read 9 Done Sent Done false
177592023-12-10T18:56:50.809Z 29 Acked 1028 Read 4 Done Sent Done false
177602023-12-10T18:56:50.809Z 30 Acked 1029 Write 6 Sent Sent Sent false
177612023-12-10T18:56:50.809Z STATES DS:0 DS:1 DS:2 TOTAL
177622023-12-10T18:56:50.809Z New 0 0 0 0
177632023-12-10T18:56:50.809Z Sent 1 30 1 32
177642023-12-10T18:56:50.809Z Done 29 0 29 58
177652023-12-10T18:56:50.809Z Skipped 0 0 0 0
177662023-12-10T18:56:50.809Z Error 0 0 0 0
177672023-12-10T18:56:50.810ZLast Flush: 1020 0 1020
177682023-12-10T18:56:50.810ZDownstairs last five completed:
177692023-12-10T18:56:50.810ZUpstairs last five completed: 30 29 28 27 26 25
177702023-12-10T18:56:50.810ZWrote out file "/tmp/test_repair_verify.data"
177712023-12-10T18:56:50.813Z
177722023-12-10T18:56:50.816ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
177732023-12-10T18:56:50.884ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
177742023-12-10T18:56:50.884Z 0 000-019 143 143 143 249 249 249 F F F
177752023-12-10T18:56:50.891Z 1 020-039 143 143 143 249 249 249 F F F
177762023-12-10T18:56:50.891Z 2 040-059 140 140 140 240 240 240 F F F
177772023-12-10T18:56:50.891Z 3 060-079 140 140 140 242 242 242 F F F
177782023-12-10T18:56:50.891Z 4 080-099 141 141 141 244 244 244 F F F
177792023-12-10T18:56:50.891Z 5 100-119 130 130 130 226 226 226 T F T <---
177802023-12-10T18:56:50.891Z 6 120-139 136 136 136 235 235 235 F F F
177812023-12-10T18:56:50.891Z 7 140-159 141 141 141 244 244 244 T F T <---
177822023-12-10T18:56:50.892Z 8 160-179 144 133 144 250 231 250 F F F <---
177832023-12-10T18:56:50.892Z 9 180-199 142 142 142 245 245 245 F F F
177842023-12-10T18:56:50.892Z 10 200-219 142 142 142 245 245 245 T F T <---
177852023-12-10T18:56:50.892Z 11 220-239 144 142 144 251 247 251 F F F <---
177862023-12-10T18:56:50.892Z 12 240-259 143 143 143 249 249 249 F F F
177872023-12-10T18:56:50.892Z 13 260-279 140 140 140 243 243 243 F F F
177882023-12-10T18:56:50.892Z 14 280-299 144 139 144 251 239 251 F F F <---
177892023-12-10T18:56:50.892Z 15 300-319 144 139 144 251 239 251 F F F <---
177902023-12-10T18:56:50.892Z 16 320-339 144 142 144 251 248 251 F F F <---
177912023-12-10T18:56:50.892Z 17 340-359 144 143 144 251 249 251 F F F <---
177922023-12-10T18:56:50.892Z 18 360-379 140 140 140 243 243 243 F F F
177932023-12-10T18:56:50.892Z 19 380-399 143 143 143 249 249 249 T F T <---
177942023-12-10T18:56:50.892Z 20 400-419 143 143 143 249 249 249 F F F
177952023-12-10T18:56:50.892Z 21 420-439 138 138 138 238 238 238 F F F
177962023-12-10T18:56:50.892Z 22 440-459 142 142 142 245 245 245 F F F
177972023-12-10T18:56:50.892Z 23 460-479 134 134 134 233 233 233 F F F
177982023-12-10T18:56:50.892Z 24 480-499 142 142 142 245 245 245 F F F
177992023-12-10T18:56:50.892Z 25 500-519 140 140 140 240 240 240 F F F
178002023-12-10T18:56:50.893Z 26 520-539 144 142 144 251 246 251 F F F <---
178012023-12-10T18:56:50.893Z 27 540-559 140 140 140 243 243 243 F F F
178022023-12-10T18:56:50.893Z 28 560-579 144 142 144 250 248 250 F F F <---
178032023-12-10T18:56:50.893Z 29 580-599 132 132 132 230 230 230 F F F
178042023-12-10T18:56:50.893ZMax gen: 144, Max flush: 251
178052023-12-10T18:56:50.893ZError: Difference in extent metadata found!
178062023-12-10T18:56:50.893ZA Difference in extent metadata is expected here
178072023-12-10T18:56:50.893ZOn loop 72
178082023-12-10T18:56:50.893Z
178092023-12-10T18:56:50.893ZVerifying data now
178102023-12-10T18:56:56.017ZNo change after verify
178112023-12-10T18:56:56.017ZLoop: 72 Downstairs dump after verify (and repair):
178122023-12-10T18:56:56.041ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
178132023-12-10T18:56:56.041Z 0 000-019 143 143 143 249 249 249 F F F
178142023-12-10T18:56:56.041Z 1 020-039 143 143 143 249 249 249 F F F
178152023-12-10T18:56:56.041Z 2 040-059 140 140 140 240 240 240 F F F
178162023-12-10T18:56:56.044Z 3 060-079 140 140 140 242 242 242 F F F
178172023-12-10T18:56:56.044Z 4 080-099 141 141 141 244 244 244 F F F
178182023-12-10T18:56:56.045Z 5 100-119 145 145 145 252 252 252 F F F
178192023-12-10T18:56:56.045Z 6 120-139 136 136 136 235 235 235 F F F
178202023-12-10T18:56:56.045Z 7 140-159 145 145 145 252 252 252 F F F
178212023-12-10T18:56:56.045Z 8 160-179 144 144 144 250 250 250 F F F
178222023-12-10T18:56:56.045Z 9 180-199 142 142 142 245 245 245 F F F
178232023-12-10T18:56:56.045Z 10 200-219 145 145 145 252 252 252 F F F
178242023-12-10T18:56:56.045Z 11 220-239 144 144 144 251 251 251 F F F
178252023-12-10T18:56:56.045Z 12 240-259 143 143 143 249 249 249 F F F
178262023-12-10T18:56:56.045Z 13 260-279 140 140 140 243 243 243 F F F
178272023-12-10T18:56:56.045Z 14 280-299 144 144 144 251 251 251 F F F
178282023-12-10T18:56:56.045Z 15 300-319 144 144 144 251 251 251 F F F
178292023-12-10T18:56:56.045Z 16 320-339 144 144 144 251 251 251 F F F
178302023-12-10T18:56:56.045Z 17 340-359 144 144 144 251 251 251 F F F
178312023-12-10T18:56:56.045Z 18 360-379 140 140 140 243 243 243 F F F
178322023-12-10T18:56:56.045Z 19 380-399 145 145 145 252 252 252 F F F
178332023-12-10T18:56:56.045Z 20 400-419 143 143 143 249 249 249 F F F
178342023-12-10T18:56:56.045Z 21 420-439 138 138 138 238 238 238 F F F
178352023-12-10T18:56:56.046Z 22 440-459 142 142 142 245 245 245 F F F
178362023-12-10T18:56:56.046Z 23 460-479 134 134 134 233 233 233 F F F
178372023-12-10T18:56:56.046Z 24 480-499 142 142 142 245 245 245 F F F
178382023-12-10T18:56:56.046Z 25 500-519 140 140 140 240 240 240 F F F
178392023-12-10T18:56:56.046Z 26 520-539 144 144 144 251 251 251 F F F
178402023-12-10T18:56:56.046Z 27 540-559 140 140 140 243 243 243 F F F
178412023-12-10T18:56:56.046Z 28 560-579 144 144 144 250 250 250 F F F
178422023-12-10T18:56:56.046Z 29 580-599 132 132 132 230 230 230 F F F
178432023-12-10T18:56:56.046ZMax gen: 145, Max flush: 252
178442023-12-10T18:56:56.049Z
178452023-12-10T18:56:56.049ZBegin loop 73
178462023-12-10T18:56:56.049ZDownstairs to restart: 2
178472023-12-10T18:56:56.080Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.08302948Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746}
178482023-12-10T18:56:56.081Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.083444771Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746}
178492023-12-10T18:56:56.087Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.083516001Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746}
178502023-12-10T18:56:56.087Z{"msg":"Crucible d065cd03-739f-4e3e-afeb-651e5eee4166 has session id: 7363e806-5f04-45dd-b1ee-50d646e5e1a2","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.083674398Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178512023-12-10T18:56:56.087Z{"msg":"Upstairs opts: Upstairs UUID: d065cd03-739f-4e3e-afeb-651e5eee4166, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.083805995Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178522023-12-10T18:56:56.087Z{"msg":"Crucible stats registered with UUID: d065cd03-739f-4e3e-afeb-651e5eee4166","v":0,"name":"crucible","level":Crucible runtime is spawned
178532023-12-10T18:56:56.087Z30,"time":"2023-12-10T18:56:56.083919602Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178542023-12-10T18:56:56.087Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30The guest has requested activation with gen:146
178552023-12-10T18:56:56.088Z,"time":"2023-12-10T18:56:56.083962138Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2","task":"up_listen"}
178562023-12-10T18:56:56.088Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.084072684Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178572023-12-10T18:56:56.088Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.084152102Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"looper":"1","session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178582023-12-10T18:56:56.088Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.084309301Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178592023-12-10T18:56:56.088Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.084350467Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"looper":"2","session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178602023-12-10T18:56:56.088Z{"msg":"[1] d065cd03-739f-4e3e-afeb-651e5eee4166 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.084418224Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"looper":"1","session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178612023-12-10T18:56:56.088Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.084458459Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"looper":"0","session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178622023-12-10T18:56:56.088Z{"msg":"[2] d065cd03-739f-4e3e-afeb-651e5eee4166 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.084526009Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"looper":"2","session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178632023-12-10T18:56:56.088Z{"msg":"[0] d065cd03-739f-4e3e-afeb-651e5eee4166 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.084566977Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"looper":"0","session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178642023-12-10T18:56:56.088Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.084886866Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178652023-12-10T18:56:56.088Z{"msg":"d065cd03-739f-4e3e-afeb-651e5eee4166 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.084950975Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178662023-12-10T18:56:56.088Z{"msg":"Set desired generation to :146","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.084976938Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178672023-12-10T18:56:56.088Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.085282579Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178682023-12-10T18:56:56.088Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.085341075Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178692023-12-10T18:56:56.088Z{"msg":"[1] d065cd03-739f-4e3e-afeb-651e5eee4166 (7363e806-5f04-45dd-b1ee-50d646e5e1a2) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.085618734Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178702023-12-10T18:56:56.089Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.085750856Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178712023-12-10T18:56:56.089Z{"msg":"[1] client is_active_req TRUE, promote! session 7363e806-5f04-45dd-b1ee-50d646e5e1a2","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.085878857Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178722023-12-10T18:56:56.089Z{"msg":"[0] d065cd03-739f-4e3e-afeb-651e5eee4166 (7363e806-5f04-45dd-b1ee-50d646e5e1a2) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.085930837Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178732023-12-10T18:56:56.089Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.085954374Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178742023-12-10T18:56:56.089Z{"msg":"[0] client is_active_req TRUE, promote! session 7363e806-5f04-45dd-b1ee-50d646e5e1a2","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.085976253Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178752023-12-10T18:56:56.089Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.086301325Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178762023-12-10T18:56:56.089Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.086393168Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178772023-12-10T18:56:56.089Z{"msg":"d065cd03-739f-4e3e-afeb-651e5eee4166 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.086487631Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178782023-12-10T18:56:56.089Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.086533217Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178792023-12-10T18:56:56.089Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.086579393Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178802023-12-10T18:56:56.089Z{"msg":"d065cd03-739f-4e3e-afeb-651e5eee4166 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.086663159Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178812023-12-10T18:56:56.089Z{"msg":"[1] d065cd03-739f-4e3e-afeb-651e5eee4166 (7363e806-5f04-45dd-b1ee-50d646e5e1a2) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.08713558Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178822023-12-10T18:56:56.089Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.087196506Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178832023-12-10T18:56:56.089Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:56.087221853Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178842023-12-10T18:56:56.089Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.087242666Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178852023-12-10T18:56:56.089Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.087392715Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178862023-12-10T18:56:56.089Z{"msg":"d065cd03-739f-4e3e-afeb-651e5eee4166 WaitActive WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.087474475Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178872023-12-10T18:56:56.089Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.087545215Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178882023-12-10T18:56:56.090Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.087617655Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178892023-12-10T18:56:56.090Z{"msg":"[0] d065cd03-739f-4e3e-afeb-651e5eee4166 (7363e806-5f04-45dd-b1ee-50d646e5e1a2) WaitActive WaitQuorum New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.08769601Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178902023-12-10T18:56:56.090Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.087792603Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178912023-12-10T18:56:56.095Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:56.087819978Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178922023-12-10T18:56:56.096Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.087861101Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178932023-12-10T18:56:56.096Z{"msg":"[2] d065cd03-739f-4e3e-afeb-651e5eee4166 (7363e806-5f04-45dd-b1ee-50d646e5e1a2) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.087886176Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178942023-12-10T18:56:56.096Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.087926593Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178952023-12-10T18:56:56.096Z{"msg":"[2] client is_active_req TRUE, promote! session 7363e806-5f04-45dd-b1ee-50d646e5e1a2","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.087950629Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178962023-12-10T18:56:56.096Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.087971535Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178972023-12-10T18:56:56.096Z{"msg":"d065cd03-739f-4e3e-afeb-651e5eee4166 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.087992619Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178982023-12-10T18:56:56.096Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.088012619Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
178992023-12-10T18:56:56.096Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.088034233Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179002023-12-10T18:56:56.096Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.088443255Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179012023-12-10T18:56:56.096Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.088498617Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179022023-12-10T18:56:56.096Z{"msg":"d065cd03-739f-4e3e-afeb-651e5eee4166 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.08852339Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179032023-12-10T18:56:56.096Z{"msg":"[2] d065cd03-739f-4e3e-afeb-651e5eee4166 (7363e806-5f04-45dd-b1ee-50d646e5e1a2) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.089177928Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179042023-12-10T18:56:56.096Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.089266514Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179052023-12-10T18:56:56.096Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:56:56.089297186Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179062023-12-10T18:56:56.096Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.089353337Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179072023-12-10T18:56:56.096Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.089380219Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179082023-12-10T18:56:56.097Z{"msg":"d065cd03-739f-4e3e-afeb-651e5eee4166 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:146
179092023-12-10T18:56:56.097ZWait for a query_work_queue command to finish before sending IO
179102023-12-10T18:56:56.097Z,"time":"2023-12-10T18:56:56.089437916Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179112023-12-10T18:56:56.097Z{"msg":"[0]R flush_numbers[0..12]: [249, 249, 240, 242, 244, 252, 235, 252, 250, 245, 252, 251]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.0894773Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179122023-12-10T18:56:56.097Z{"msg":"[0]R generation[0..12]: [143, 143, 140, 140, 141, 145, 136, 145, 144, 142, 145, 144]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.089512424Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179132023-12-10T18:56:56.097Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.089536146Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179142023-12-10T18:56:56.097Z{"msg":"[1]R flush_numbers[0..12]: [249, 249, 240, 242, 244, 252, 235, 252, 250, 245, 252, 251]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.089607715Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179152023-12-10T18:56:56.097Z{"msg":"[1]R generation[0..12]: [143, 143, 140, 140, 141, 145, 136, 145, 144, 142, 145, 144]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.08963453Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179162023-12-10T18:56:56.097Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.089662915Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179172023-12-10T18:56:56.097Z{"msg":"[2]R flush_numbers[0..12]: [249, 249, 240, 242, 244, 252, 235, 252, 250, 245, 252, 251]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.089718909Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179182023-12-10T18:56:56.097Z{"msg":"[2]R generation[0..12]: [143, 143, 140, 140, 141, 145, 136, 145, 144, 142, 145, 144]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.089744739Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179192023-12-10T18:56:56.097Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.089823309Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179202023-12-10T18:56:56.097Z{"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.089849519Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179212023-12-10T18:56:56.097Z{"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.08991891Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179222023-12-10T18:56:56.097Z{"msg":"Next flush: 253","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.089944683Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179232023-12-10T18:56:56.097Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.089964587Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179242023-12-10T18:56:56.097Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.090039873Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179252023-12-10T18:56:56.098Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.090065701Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179262023-12-10T18:56:56.098Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.090090893Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179272023-12-10T18:56:56.098Z{"msg":"d065cd03-739f-4e3e-afeb-651e5eee4166 is now active with session: 7363e806-5f04-45dd-b1ee-50d646e5e1a2","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.090146836Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179282023-12-10T18:56:56.098Z{"msg":"d065cd03-739f-4e3e-afeb-651e5eee4166 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.090172168Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179292023-12-10T18:56:56.098Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
179302023-12-10T18:56:56.099Z,"time":"2023-12-10T18:56:56.090197835Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179312023-12-10T18:56:56.099Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.09029792Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179322023-12-10T18:56:56.099Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.090357317Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179332023-12-10T18:56:56.099Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.090382512Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179342023-12-10T18:56:56.099Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.090406759Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179352023-12-10T18:56:56.099Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.090461574Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179362023-12-10T18:56:56.099Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.09048669Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179372023-12-10T18:56:56.099Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.090542604Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179382023-12-10T18:56:56.099Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.090570088Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179392023-12-10T18:56:56.099Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.090624252Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179402023-12-10T18:56:56.099Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:56:56.090652242Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1746,"session_id":"7363e806-5f04-45dd-b1ee-50d646e5e1a2"}
179412023-12-10T18:56:56.099ZLoading write count information from file "/tmp/test_repair_verify.data"
179422023-12-10T18:56:56.099ZRun Repair workload
179432023-12-10T18:56:56.099Z01/30 Write block 582 len 40960 data: 9 8 9 7 9 10 11 12 14 13
179442023-12-10T18:56:56.099Z02/30 Write block 307 len 12288 data: 10 8 8
179452023-12-10T18:56:56.099Z03/30 Write block 310 len 40960 data: 6 6 7 8 9 9 7 6 9 10
179462023-12-10T18:56:56.100Z04/30 Write block 36 len 28672 data: 14 13 12 11 10 9 9
179472023-12-10T18:56:56.100Z05/30 Read block 205 len 28672
179482023-12-10T18:56:56.103Z06/30 Read block 186 len 32768
179492023-12-10T18:56:56.103Z07/30 Read block 516 len 12288
179502023-12-10T18:56:56.103Z08/30 Read block 317 len 32768
179512023-12-10T18:56:56.106Z09/30 Write block 36 len 4096 data: 15
179522023-12-10T18:56:56.106Z10/30 Read block 312 len 40960
179532023-12-10T18:56:56.109Z11/30 Read block 598 len 4096
179542023-12-10T18:56:56.109Z12/30 Read block 278 len 4096
179552023-12-10T18:56:56.112Z13/30 Write block 460 len 40960 data: 9 8 10 10 9 12 9 11 10 13
179562023-12-10T18:56:56.112Z14/30 Write block 286 len 8192 data: 14 14
179572023-12-10T18:56:56.112Z15/30 Write block 308 len 4096 data: 9
179582023-12-10T18:56:56.115Z16/30 Write block 69 len 8192 data: 15 13
179592023-12-10T18:56:56.115Z17/30 Flush
179602023-12-10T18:56:56.118Z18/30 Flush
179612023-12-10T18:56:56.121Z19/30 Write block 559 len 36864 data: 10 10 12 14 11 12 12 13 13
179622023-12-10T18:56:56.122Z20/30 Flush
179632023-12-10T18:56:56.124Z21/30 Write block 5 len 36864 data: 4 5 9 11 11 13 11 14 16
179642023-12-10T18:56:56.124Z22/30 Read block 550 len 4096
179652023-12-10T18:56:56.130Z23/30 Read block 528 len 4096
179662023-12-10T18:56:56.130Z24/30 Read block 71 len 36864
179672023-12-10T18:56:56.138Z25/30 Read block 93 len 36864
179682023-12-10T18:56:56.138Z26/30 Read block 299 len 20480
179692023-12-10T18:56:56.138Z27/30 Read block 312 len 12288
179702023-12-10T18:56:56.138Z28/30 Read block 291 len 28672
179712023-12-10T18:56:56.142Z29/30 Write block 324 len 28672 data: 12 10 12 13 12 11 12
179722023-12-10T18:56:56.142Z30/30 Write block 355 len 36864 data: 12 14 12 12 13 11 10 9 9
179732023-12-10T18:56:56.142Z----------------------------------------------------------------
179742023-12-10T18:56:56.142Z Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:30
179752023-12-10T18:56:56.142ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
179762023-12-10T18:56:56.142Z 1 Acked 1000 Write 10 Done Done Sent false
179772023-12-10T18:56:56.142Z 2 Acked 1001 Write 3 Done Done Sent false
179782023-12-10T18:56:56.142Z 3 Acked 1002 Write 10 Done Done Sent false
179792023-12-10T18:56:56.143Z 4 Acked 1003 Write 7 Done Done Sent false
179802023-12-10T18:56:56.143Z 5 Acked 1004 Read 7 Done Done Sent false
179812023-12-10T18:56:56.143Z 6 Acked 1005 Read 8 Done Done Sent false
179822023-12-10T18:56:56.143Z 7 Acked 1006 Read 3 Done Done Sent false
179832023-12-10T18:56:56.143Z 8 Acked 1007 Read 8 Done Done Sent false
179842023-12-10T18:56:56.143Z 9 Acked 1008 Write 1 Done Done Sent false
179852023-12-10T18:56:56.143Z 10 Acked 1009 Read 10 Done Done Sent false
179862023-12-10T18:56:56.143Z 11 Acked 1010 Read 1 Done Done Sent false
179872023-12-10T18:56:56.144Z 12 Acked 1011 Read 1 Done Done Sent false
179882023-12-10T18:56:56.144Z 13 Acked 1012 Write 10 Done Done Sent false
179892023-12-10T18:56:56.144Z 14 Acked 1013 Write 2 Done Done Sent false
179902023-12-10T18:56:56.144Z 15 Acked 1014 Write 1 Done Done Sent false
179912023-12-10T18:56:56.144Z 16 Acked 1015 Write 2 Done Done Sent false
179922023-12-10T18:56:56.144Z 17 Acked 1016 Flush 0 Done Done Sent false
179932023-12-10T18:56:56.144Z 18 Acked 1017 Flush 0 Done Done Sent false
179942023-12-10T18:56:56.144Z 19 Acked 1018 Write 9 Done Done Sent false
179952023-12-10T18:56:56.144Z 20 Acked 1019 Flush 0 Done Done Sent false
179962023-12-10T18:56:56.144Z 21 Acked 1020 Write 9 Done Done Sent false
179972023-12-10T18:56:56.144Z 22 Acked 1021 Read 1 Done Done Sent false
179982023-12-10T18:56:56.144Z 23 Acked 1022 Read 1 Done Done Sent false
179992023-12-10T18:56:56.144Z 24 Acked 1023 Read 9 Done Done Sent false
180002023-12-10T18:56:56.144Z 25 Acked 1024 Read 9 Done Done Sent false
180012023-12-10T18:56:56.144Z 26 Acked 1025 Read 5 Done Done Sent false
180022023-12-10T18:56:56.144Z 27 Acked 1026 Read 3 Done Done Sent false
180032023-12-10T18:56:56.144Z 28 Acked 1027 Read 7 Done Done Sent false
180042023-12-10T18:56:56.144Z 29 Acked 1028 Write 7 Done Sent Sent false
180052023-12-10T18:56:56.145Z 30 Acked 1029 Write 9 New New New false
180062023-12-10T18:56:56.145Z STATES DS:0 DS:1 DS:2 TOTAL
180072023-12-10T18:56:56.145Z New 1 1 1 3
180082023-12-10T18:56:56.145Z Sent 0 1 29 30
180092023-12-10T18:56:56.145Z Done 29 28 0 57
180102023-12-10T18:56:56.145Z Skipped 0 0 0 0
180112023-12-10T18:56:56.145Z Error 0 0 0 0
180122023-12-10T18:56:56.145ZLast Flush: 1019 1019 0
180132023-12-10T18:56:56.145ZDownstairs last five completed:
180142023-12-10T18:56:56.145ZUpstairs last five completed: 30 29 28 27 26 25
180152023-12-10T18:56:56.145ZWrote out file "/tmp/test_repair_verify.data"
180162023-12-10T18:56:56.148Z
180172023-12-10T18:56:56.154ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
180182023-12-10T18:56:56.179ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
180192023-12-10T18:56:56.179Z 0 000-019 143 143 143 249 249 249 T T F <---
180202023-12-10T18:56:56.183Z 1 020-039 146 146 143 253 253 249 F F F <---
180212023-12-10T18:56:56.183Z 2 040-059 146 146 140 253 253 240 F F F <---
180222023-12-10T18:56:56.183Z 3 060-079 146 146 140 253 253 242 F F F <---
180232023-12-10T18:56:56.183Z 4 080-099 141 141 141 244 244 244 F F F
180242023-12-10T18:56:56.183Z 5 100-119 145 145 145 252 252 252 F F F
180252023-12-10T18:56:56.183Z 6 120-139 136 136 136 235 235 235 F F F
180262023-12-10T18:56:56.183Z 7 140-159 145 145 145 252 252 252 F F F
180272023-12-10T18:56:56.183Z 8 160-179 144 144 144 250 250 250 F F F
180282023-12-10T18:56:56.183Z 9 180-199 142 142 142 245 245 245 F F F
180292023-12-10T18:56:56.183Z 10 200-219 145 145 145 252 252 252 F F F
180302023-12-10T18:56:56.183Z 11 220-239 144 144 144 251 251 251 F F F
180312023-12-10T18:56:56.184Z 12 240-259 143 143 143 249 249 249 F F F
180322023-12-10T18:56:56.184Z 13 260-279 140 140 140 243 243 243 F F F
180332023-12-10T18:56:56.184Z 14 280-299 146 146 144 253 253 251 F F F <---
180342023-12-10T18:56:56.184Z 15 300-319 146 146 144 253 253 251 F F F <---
180352023-12-10T18:56:56.184Z 16 320-339 144 144 144 251 251 251 T T F <---
180362023-12-10T18:56:56.184Z 17 340-359 144 144 144 251 251 251 T T F <---
180372023-12-10T18:56:56.184Z 18 360-379 140 140 140 243 243 243 T T F <---
180382023-12-10T18:56:56.184Z 19 380-399 145 145 145 252 252 252 F F F
180392023-12-10T18:56:56.185Z 20 400-419 143 143 143 249 249 249 F F F
180402023-12-10T18:56:56.185Z 21 420-439 138 138 138 238 238 238 F F F
180412023-12-10T18:56:56.185Z 22 440-459 142 142 142 245 245 245 F F F
180422023-12-10T18:56:56.185Z 23 460-479 146 146 134 253 253 233 F F F <---
180432023-12-10T18:56:56.185Z 24 480-499 142 142 142 245 245 245 F F F
180442023-12-10T18:56:56.185Z 25 500-519 140 140 140 240 240 240 F F F
180452023-12-10T18:56:56.185Z 26 520-539 144 144 144 251 251 251 F F F
180462023-12-10T18:56:56.185Z 27 540-559 146 146 140 255 255 243 F F F <---
180472023-12-10T18:56:56.185Z 28 560-579 146 146 144 255 255 250 F F F <---
180482023-12-10T18:56:56.185Z 29 580-599 146 146 132 253 253 230 F F F <---
180492023-12-10T18:56:56.185ZMax gen: 146, Max flush: 255
180502023-12-10T18:56:56.185ZError: Difference in extent metadata found!
180512023-12-10T18:56:56.188ZA Difference in extent metadata is expected here
180522023-12-10T18:56:56.188ZOn loop 73
180532023-12-10T18:56:56.188Z
180542023-12-10T18:56:56.189ZVerifying data now
180552023-12-10T18:57:01.445ZNo change after verify
180562023-12-10T18:57:01.446ZLoop: 73 Downstairs dump after verify (and repair):
180572023-12-10T18:57:01.469ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
180582023-12-10T18:57:01.469Z 0 000-019 147 147 147 256 256 256 F F F
180592023-12-10T18:57:01.469Z 1 020-039 146 146 146 253 253 253 F F F
180602023-12-10T18:57:01.469Z 2 040-059 146 146 146 253 253 253 F F F
180612023-12-10T18:57:01.469Z 3 060-079 146 146 146 253 253 253 F F F
180622023-12-10T18:57:01.473Z 4 080-099 141 141 141 244 244 244 F F F
180632023-12-10T18:57:01.474Z 5 100-119 145 145 145 252 252 252 F F F
180642023-12-10T18:57:01.474Z 6 120-139 136 136 136 235 235 235 F F F
180652023-12-10T18:57:01.474Z 7 140-159 145 145 145 252 252 252 F F F
180662023-12-10T18:57:01.474Z 8 160-179 144 144 144 250 250 250 F F F
180672023-12-10T18:57:01.474Z 9 180-199 142 142 142 245 245 245 F F F
180682023-12-10T18:57:01.475Z 10 200-219 145 145 145 252 252 252 F F F
180692023-12-10T18:57:01.475Z 11 220-239 144 144 144 251 251 251 F F F
180702023-12-10T18:57:01.475Z 12 240-259 143 143 143 249 249 249 F F F
180712023-12-10T18:57:01.475Z 13 260-279 140 140 140 243 243 243 F F F
180722023-12-10T18:57:01.475Z 14 280-299 146 146 146 253 253 253 F F F
180732023-12-10T18:57:01.475Z 15 300-319 146 146 146 253 253 253 F F F
180742023-12-10T18:57:01.475Z 16 320-339 147 147 147 256 256 256 F F F
180752023-12-10T18:57:01.475Z 17 340-359 147 147 147 256 256 256 F F F
180762023-12-10T18:57:01.475Z 18 360-379 147 147 147 256 256 256 F F F
180772023-12-10T18:57:01.475Z 19 380-399 145 145 145 252 252 252 F F F
180782023-12-10T18:57:01.475Z 20 400-419 143 143 143 249 249 249 F F F
180792023-12-10T18:57:01.475Z 21 420-439 138 138 138 238 238 238 F F F
180802023-12-10T18:57:01.475Z 22 440-459 142 142 142 245 245 245 F F F
180812023-12-10T18:57:01.475Z 23 460-479 146 146 146 253 253 253 F F F
180822023-12-10T18:57:01.475Z 24 480-499 142 142 142 245 245 245 F F F
180832023-12-10T18:57:01.475Z 25 500-519 140 140 140 240 240 240 F F F
180842023-12-10T18:57:01.475Z 26 520-539 144 144 144 251 251 251 F F F
180852023-12-10T18:57:01.475Z 27 540-559 146 146 146 255 255 255 F F F
180862023-12-10T18:57:01.476Z 28 560-579 146 146 146 255 255 255 F F F
180872023-12-10T18:57:01.476Z 29 580-599 146 146 146 253 253 253 F F F
180882023-12-10T18:57:01.476ZMax gen: 147, Max flush: 256
180892023-12-10T18:57:01.480Z
180902023-12-10T18:57:01.480ZBegin loop 74
180912023-12-10T18:57:01.480ZDownstairs to restart: 0
180922023-12-10T18:57:01.509Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.511725876Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754}
180932023-12-10T18:57:01.510Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.512153729Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754}
180942023-12-10T18:57:01.510Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.512182704Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754}
180952023-12-10T18:57:01.516Z{"msg":"Crucible b270ed50-4c99-40c6-8a7a-3d37e108c5b9 has session id: 35ef1121-5f16-4696-bf51-dc3448f91305","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.5122522Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
180962023-12-10T18:57:01.516Z{"msg":"Upstairs opts: Upstairs UUID: b270ed50-4c99-40c6-8a7a-3d37e108c5b9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.51231809Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
180972023-12-10T18:57:01.516Z{"msg":"Crucible stats registered with UUID: b270ed50-4c99-40c6-8a7a-3d37e108c5b9","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.512360458Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
180982023-12-10T18:57:01.516ZCrucible runtime is spawned
180992023-12-10T18:57:01.516ZThe guest has requested activation with gen:148
181002023-12-10T18:57:01.516Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.51271815Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305","task":"up_listen"}
181012023-12-10T18:57:01.516Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.512820751Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181022023-12-10T18:57:01.516Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.512907596Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181032023-12-10T18:57:01.517Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.512949382Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"looper":"1","session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181042023-12-10T18:57:01.517Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.512997764Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"looper":"2","session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181052023-12-10T18:57:01.517Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.513039126Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"looper":"0","session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181062023-12-10T18:57:01.517Z{"msg":"[0] b270ed50-4c99-40c6-8a7a-3d37e108c5b9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.513081689Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"looper":"0","session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181072023-12-10T18:57:01.517Z{"msg":"[1] b270ed50-4c99-40c6-8a7a-3d37e108c5b9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.513216109Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"looper":"1","session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181082023-12-10T18:57:01.517Z{"msg":"[2] b270ed50-4c99-40c6-8a7a-3d37e108c5b9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.513260317Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"looper":"2","session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181092023-12-10T18:57:01.517Z{"msg":"b270ed50-4c99-40c6-8a7a-3d37e108c5b9 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.513300959Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181102023-12-10T18:57:01.517Z{"msg":"Set desired generation to :148","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.513338821Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181112023-12-10T18:57:01.517Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.51337683Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181122023-12-10T18:57:01.518Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.513691448Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181132023-12-10T18:57:01.518Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.513755563Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181142023-12-10T18:57:01.518Z{"msg":"[1] b270ed50-4c99-40c6-8a7a-3d37e108c5b9 (35ef1121-5f16-4696-bf51-dc3448f91305) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.514212353Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181152023-12-10T18:57:01.518Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.514276298Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181162023-12-10T18:57:01.518Z{"msg":"[1] client is_active_req TRUE, promote! session 35ef1121-5f16-4696-bf51-dc3448f91305","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.514301866Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181172023-12-10T18:57:01.518Z{"msg":"[2] b270ed50-4c99-40c6-8a7a-3d37e108c5b9 (35ef1121-5f16-4696-bf51-dc3448f91305) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.514460951Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181182023-12-10T18:57:01.518Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.514568514Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181192023-12-10T18:57:01.518Z{"msg":"[2] client is_active_req TRUE, promote! session 35ef1121-5f16-4696-bf51-dc3448f91305","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.51464127Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181202023-12-10T18:57:01.518Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.514968368Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181212023-12-10T18:57:01.518Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.515039169Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181222023-12-10T18:57:01.519Z{"msg":"b270ed50-4c99-40c6-8a7a-3d37e108c5b9 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.515066952Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181232023-12-10T18:57:01.520Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.515222053Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181242023-12-10T18:57:01.520Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.515411922Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181252023-12-10T18:57:01.520Z{"msg":"b270ed50-4c99-40c6-8a7a-3d37e108c5b9 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.515455602Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181262023-12-10T18:57:01.520Z{"msg":"[1] b270ed50-4c99-40c6-8a7a-3d37e108c5b9 (35ef1121-5f16-4696-bf51-dc3448f91305) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.515910458Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181272023-12-10T18:57:01.520Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.516030757Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181282023-12-10T18:57:01.520Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:01.516159499Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181292023-12-10T18:57:01.526Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.516202401Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181302023-12-10T18:57:01.526Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.516268598Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181312023-12-10T18:57:01.526Z{"msg":"b270ed50-4c99-40c6-8a7a-3d37e108c5b9 New WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.516339765Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181322023-12-10T18:57:01.526Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.516402226Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181332023-12-10T18:57:01.526Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.516472451Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181342023-12-10T18:57:01.526Z{"msg":"[2] b270ed50-4c99-40c6-8a7a-3d37e108c5b9 (35ef1121-5f16-4696-bf51-dc3448f91305) New WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.516514602Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181352023-12-10T18:57:01.526Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.516578055Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181362023-12-10T18:57:01.526Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:01.516639813Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181372023-12-10T18:57:01.527Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.516664633Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181382023-12-10T18:57:01.527Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.516705852Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181392023-12-10T18:57:01.527Z{"msg":"b270ed50-4c99-40c6-8a7a-3d37e108c5b9 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.51672869Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181402023-12-10T18:57:01.527Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.51676926Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181412023-12-10T18:57:01.527Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.516791008Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181422023-12-10T18:57:01.528Z{"msg":"[0] b270ed50-4c99-40c6-8a7a-3d37e108c5b9 (35ef1121-5f16-4696-bf51-dc3448f91305) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.516833605Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181432023-12-10T18:57:01.528Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.516856253Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181442023-12-10T18:57:01.528Z{"msg":"[0] client is_active_req TRUE, promote! session 35ef1121-5f16-4696-bf51-dc3448f91305","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.516897928Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181452023-12-10T18:57:01.528Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.517461966Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181462023-12-10T18:57:01.528Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.517520284Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181472023-12-10T18:57:01.528Z{"msg":"b270ed50-4c99-40c6-8a7a-3d37e108c5b9 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.517545901Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181482023-12-10T18:57:01.528Z{"msg":"[0] b270ed50-4c99-40c6-8a7a-3d37e108c5b9 (35ef1121-5f16-4696-bf51-dc3448f91305) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.518121062Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181492023-12-10T18:57:01.528Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.518175271Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181502023-12-10T18:57:01.528Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:01.518198647Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181512023-12-10T18:57:01.528Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.518218948Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181522023-12-10T18:57:01.528Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.518245599Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181532023-12-10T18:57:01.529Z{"msg":"b270ed50-4c99-40c6-8a7a-3d37e108c5b9 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.518293061Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181542023-12-10T18:57:01.529Z{"msg":"[0]R flush_numbers[0..12]: [256, 253, 253, 253, 244, 252, 235, 252, 250, 245, 252, 251]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.518318032Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181552023-12-10T18:57:01.529Z{"msg":"[0]R generation[0..12]: [147, 146, 146, 146, 141, 145, 136, 145, 144, 142, 145, 144]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.518339815Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181562023-12-10T18:57:01.529Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.518396984Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181572023-12-10T18:57:01.529Z{"msg":"[1]R flush_numbers[0..12]: [256, 253, 253, 253, 244, 252, 235, 252, 250, 245, 252, 251]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.518425257Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181582023-12-10T18:57:01.529Z{"msg":"[1]R generation[0..12]: [147, 146, 146, 146, 141, 145, 136, 145, 144, 142, 145, 144]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:148
181592023-12-10T18:57:01.529Z,"time":"2023-12-10T18:57:01.518468668Z","hostname":"Wait for a query_work_queue command to finish before sending IO
181602023-12-10T18:57:01.529Zip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181612023-12-10T18:57:01.529Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.51849703Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181622023-12-10T18:57:01.529Z{"msg":"[2]R flush_numbers[0..12]: [256, 253, 253, 253, 244, 252, 235, 252, 250, 245, 252, 251]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.518599456Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181632023-12-10T18:57:01.529Z{"msg":"[2]R generation[0..12]: [147, 146, 146, 146, 141, 145, 136, 145, 144, 142, 145, 144]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.518644412Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181642023-12-10T18:57:01.529Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.518684088Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181652023-12-10T18:57:01.529Z{"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.518720938Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181662023-12-10T18:57:01.529Z{"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.518781595Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181672023-12-10T18:57:01.529Z{"msg":"Next flush: 257","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.51880663Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181682023-12-10T18:57:01.529Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.518826565Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181692023-12-10T18:57:01.529Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.5188675Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181702023-12-10T18:57:01.529Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.518891848Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181712023-12-10T18:57:01.530Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.518927155Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181722023-12-10T18:57:01.530Z{"msg":"b270ed50-4c99-40c6-8a7a-3d37e108c5b9 is now active with session: 35ef1121-5f16-4696-bf51-dc3448f91305","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.518979421Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181732023-12-10T18:57:01.530Z{"msg":"b270ed50-4c99-40c6-8a7a-3d37e108c5b9 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.519003686Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181742023-12-10T18:57:01.530Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.519044805Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181752023-12-10T18:57:01.531Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.519066809Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181762023-12-10T18:57:01.531Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.519086546Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181772023-12-10T18:57:01.531Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.519127756Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181782023-12-10T18:57:01.531Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.519161734Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181792023-12-10T18:57:01.531Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.519225656Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181802023-12-10T18:57:01.531Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.519291608Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181812023-12-10T18:57:01.531Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.51935803Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181822023-12-10T18:57:01.531Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.519448273Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181832023-12-10T18:57:01.531ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
181842023-12-10T18:57:01.531Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.519568777Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181852023-12-10T18:57:01.531Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:01.519645558Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1754,"session_id":"35ef1121-5f16-4696-bf51-dc3448f91305"}
181862023-12-10T18:57:01.531ZLoading write count information from file "/tmp/test_repair_verify.data"
181872023-12-10T18:57:01.531ZRun Repair workload
181882023-12-10T18:57:01.531Z01/30 Write block 113 len 28672 data: 13 11 12 13 12 12 12
181892023-12-10T18:57:01.531Z02/30 Read block 61 len 8192
181902023-12-10T18:57:01.531Z03/30 Read block 175 len 16384
181912023-12-10T18:57:01.531Z04/30 Write block 152 len 8192 data: 10 10
181922023-12-10T18:57:01.566Z05/30 Write block 339 len 40960 data: 11 10 11 12 16 16 14 13 16 16
181932023-12-10T18:57:01.566Z06/30 Flush
181942023-12-10T18:57:01.566Z07/30 Write block 315 len 20480 data: 10 8 7 10 11
181952023-12-10T18:57:01.566Z08/30 Read block 50 len 16384
181962023-12-10T18:57:01.566Z09/30 Flush
181972023-12-10T18:57:01.566Z10/30 Write block 399 len 4096 data: 12
181982023-12-10T18:57:01.566Z11/30 Read block 530 len 28672
181992023-12-10T18:57:01.571Z12/30 Write block 528 len 20480 data: 11 14 12 15 14
182002023-12-10T18:57:01.571Z13/30 Write block 92 len 12288 data: 11 13 13
182012023-12-10T18:57:01.571Z14/30 Write block 540 len 24576 data: 16 18 18 16 15 11
182022023-12-10T18:57:01.571Z15/30 Read block 535 len 20480
182032023-12-10T18:57:01.571Z16/30 Read block 279 len 16384
182042023-12-10T18:57:01.571Z17/30 Read block 427 len 12288
182052023-12-10T18:57:01.571Z18/30 Write block 366 len 28672 data: 7 7 7 8 7 10 11
182062023-12-10T18:57:01.571Z19/30 Write block 313 len 36864 data: 9 10 11 9 8 11 12 11 11
182072023-12-10T18:57:01.571Z20/30 Read block 191 len 8192
182082023-12-10T18:57:01.571Z21/30 Read block 120 len 4096
182092023-12-10T18:57:01.571Z22/30 Flush
182102023-12-10T18:57:01.571Z23/30 Write block 332 len 24576 data: 13 13 11 9 10 10
182112023-12-10T18:57:01.571Z24/30 Read block 116 len 4096
182122023-12-10T18:57:01.571Z25/30 Read block 566 len 32768
182132023-12-10T18:57:01.571Z26/30 Read block 178 len 28672
182142023-12-10T18:57:01.572Z27/30 Read block 226 len 36864
182152023-12-10T18:57:01.572Z28/30 Read block 349 len 8192
182162023-12-10T18:57:01.572Z29/30 Write block 58 len 36864 data: 6 5 7 8 10 11 12 14 15
182172023-12-10T18:57:01.572Z30/30 Read block 532 len 20480
182182023-12-10T18:57:01.576Z----------------------------------------------------------------
182192023-12-10T18:57:01.577Z Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30
182202023-12-10T18:57:01.577ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
182212023-12-10T18:57:01.577Z 1 Acked 1000 Write 7 Sent Done Done false
182222023-12-10T18:57:01.577Z 2 Acked 1001 Read 2 Sent Done Done false
182232023-12-10T18:57:01.577Z 3 Acked 1002 Read 4 Sent Done Done false
182242023-12-10T18:57:01.577Z 4 Acked 1003 Write 2 Sent Done Done false
182252023-12-10T18:57:01.577Z 5 Acked 1004 Write 10 Sent Done Done false
182262023-12-10T18:57:01.577Z 6 Acked 1005 Flush 0 Sent Done Done false
182272023-12-10T18:57:01.577Z 7 Acked 1006 Write 5 Sent Done Done false
182282023-12-10T18:57:01.577Z 8 Acked 1007 Read 4 Sent Done Done false
182292023-12-10T18:57:01.577Z 9 Acked 1008 Flush 0 Sent Done Done false
182302023-12-10T18:57:01.577Z 10 Acked 1009 Write 1 Sent Done Done false
182312023-12-10T18:57:01.577Z 11 Acked 1010 Read 7 Sent Done Done false
182322023-12-10T18:57:01.577Z 12 Acked 1011 Write 5 Sent Done Done false
182332023-12-10T18:57:01.577Z 13 Acked 1012 Write 3 Sent Done Done false
182342023-12-10T18:57:01.578Z 14 Acked 1013 Write 6 Sent Done Done false
182352023-12-10T18:57:01.578Z 15 Acked 1014 Read 5 Sent Done Done false
182362023-12-10T18:57:01.578Z 16 Acked 1015 Read 4 Sent Done Done false
182372023-12-10T18:57:01.578Z 17 Acked 1016 Read 3 Sent Done Done false
182382023-12-10T18:57:01.578Z 18 Acked 1017 Write 7 Sent Done Done false
182392023-12-10T18:57:01.578Z 19 Acked 1018 Write 9 Sent Done Done false
182402023-12-10T18:57:01.578Z 20 Acked 1019 Read 2 Sent Done Done false
182412023-12-10T18:57:01.578Z 21 Acked 1020 Read 1 Sent Done Done false
182422023-12-10T18:57:01.578Z 22 Acked 1021 Flush 0 Sent Done Done false
182432023-12-10T18:57:01.578Z 23 Acked 1022 Write 6 Sent Done Done false
182442023-12-10T18:57:01.578Z 24 Acked 1023 Read 1 Sent Done Done false
182452023-12-10T18:57:01.578Z 25 Acked 1024 Read 8 Sent Done Done false
182462023-12-10T18:57:01.578Z 26 Acked 1025 Read 7 Sent Done Done false
182472023-12-10T18:57:01.578Z 27 Acked 1026 Read 9 Sent Done Done false
182482023-12-10T18:57:01.578Z 28 Acked 1027 Read 2 Sent Done Done false
182492023-12-10T18:57:01.578Z 29 Acked 1028 Write 9 Sent Done Done false
182502023-12-10T18:57:01.578Z 30 Acked 1029 Read 5 Sent Done Done false
182512023-12-10T18:57:01.578Z STATES DS:0 DS:1 DS:2 TOTAL
182522023-12-10T18:57:01.579Z New 0 0 0 0
182532023-12-10T18:57:01.579Z Sent 30 0 0 30
182542023-12-10T18:57:01.579Z Done 0 30 30 60
182552023-12-10T18:57:01.579Z Skipped 0 0 0 0
182562023-12-10T18:57:01.579Z Error 0 0 0 0
182572023-12-10T18:57:01.579ZLast Flush: 0 1021 1021
182582023-12-10T18:57:01.579ZDownstairs last five completed:
182592023-12-10T18:57:01.579ZUpstairs last five completed: 30 29 28 27 26 25
182602023-12-10T18:57:01.579ZWrote out file "/tmp/test_repair_verify.data"
182612023-12-10T18:57:01.583Z
182622023-12-10T18:57:01.586ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
182632023-12-10T18:57:01.610ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
182642023-12-10T18:57:01.610Z 0 000-019 147 147 147 256 256 256 F F F
182652023-12-10T18:57:01.614Z 1 020-039 146 146 146 253 253 253 F F F
182662023-12-10T18:57:01.614Z 2 040-059 146 146 146 253 253 253 F T T <---
182672023-12-10T18:57:01.614Z 3 060-079 146 146 146 253 253 253 F T T <---
182682023-12-10T18:57:01.614Z 4 080-099 141 148 148 244 259 259 F F F <---
182692023-12-10T18:57:01.614Z 5 100-119 145 148 148 252 257 257 F F F <---
182702023-12-10T18:57:01.614Z 6 120-139 136 136 136 235 235 235 F F F
182712023-12-10T18:57:01.614Z 7 140-159 145 148 148 252 257 257 F F F <---
182722023-12-10T18:57:01.614Z 8 160-179 144 144 144 250 250 250 F F F
182732023-12-10T18:57:01.614Z 9 180-199 142 142 142 245 245 245 F F F
182742023-12-10T18:57:01.614Z 10 200-219 145 145 145 252 252 252 F F F
182752023-12-10T18:57:01.614Z 11 220-239 144 144 144 251 251 251 F F F
182762023-12-10T18:57:01.614Z 12 240-259 143 143 143 249 249 249 F F F
182772023-12-10T18:57:01.614Z 13 260-279 140 140 140 243 243 243 F F F
182782023-12-10T18:57:01.614Z 14 280-299 146 146 146 253 253 253 F F F
182792023-12-10T18:57:01.614Z 15 300-319 146 148 148 253 259 259 F F F <---
182802023-12-10T18:57:01.615Z 16 320-339 147 148 148 256 259 259 F T T <---
182812023-12-10T18:57:01.615Z 17 340-359 147 148 148 256 257 257 F F F <---
182822023-12-10T18:57:01.615Z 18 360-379 147 148 148 256 259 259 F F F <---
182832023-12-10T18:57:01.615Z 19 380-399 145 148 148 252 259 259 F F F <---
182842023-12-10T18:57:01.615Z 20 400-419 143 143 143 249 249 249 F F F
182852023-12-10T18:57:01.615Z 21 420-439 138 138 138 238 238 238 F F F
182862023-12-10T18:57:01.615Z 22 440-459 142 142 142 245 245 245 F F F
182872023-12-10T18:57:01.615Z 23 460-479 146 146 146 253 253 253 F F F
182882023-12-10T18:57:01.615Z 24 480-499 142 142 142 245 245 245 F F F
182892023-12-10T18:57:01.615Z 25 500-519 140 140 140 240 240 240 F F F
182902023-12-10T18:57:01.615Z 26 520-539 144 148 148 251 259 259 F F F <---
182912023-12-10T18:57:01.615Z 27 540-559 146 148 148 255 259 259 F F F <---
182922023-12-10T18:57:01.615Z 28 560-579 146 146 146 255 255 255 F F F
182932023-12-10T18:57:01.615Z 29 580-599 146 146 146 253 253 253 F F F
182942023-12-10T18:57:01.615ZMax gen: 148, Max flush: 259
182952023-12-10T18:57:01.615ZError: Difference in extent metadata found!
182962023-12-10T18:57:01.619ZA Difference in extent metadata is expected here
182972023-12-10T18:57:01.619ZOn loop 74
182982023-12-10T18:57:01.619Z
182992023-12-10T18:57:01.625ZVerifying data now
183002023-12-10T18:57:06.820ZNo change after verify
183012023-12-10T18:57:06.820ZLoop: 74 Downstairs dump after verify (and repair):
183022023-12-10T18:57:06.843ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
183032023-12-10T18:57:06.843Z 0 000-019 147 147 147 256 256 256 F F F
183042023-12-10T18:57:06.843Z 1 020-039 146 146 146 253 253 253 F F F
183052023-12-10T18:57:06.843Z 2 040-059 149 149 149 260 260 260 F F F
183062023-12-10T18:57:06.843Z 3 060-079 149 149 149 260 260 260 F F F
183072023-12-10T18:57:06.843Z 4 080-099 148 148 148 259 259 259 F F F
183082023-12-10T18:57:06.843Z 5 100-119 148 148 148 257 257 257 F F F
183092023-12-10T18:57:06.846Z 6 120-139 136 136 136 235 235 235 F F F
183102023-12-10T18:57:06.847Z 7 140-159 148 148 148 257 257 257 F F F
183112023-12-10T18:57:06.847Z 8 160-179 144 144 144 250 250 250 F F F
183122023-12-10T18:57:06.847Z 9 180-199 142 142 142 245 245 245 F F F
183132023-12-10T18:57:06.847Z 10 200-219 145 145 145 252 252 252 F F F
183142023-12-10T18:57:06.847Z 11 220-239 144 144 144 251 251 251 F F F
183152023-12-10T18:57:06.847Z 12 240-259 143 143 143 249 249 249 F F F
183162023-12-10T18:57:06.847Z 13 260-279 140 140 140 243 243 243 F F F
183172023-12-10T18:57:06.847Z 14 280-299 146 146 146 253 253 253 F F F
183182023-12-10T18:57:06.847Z 15 300-319 148 148 148 259 259 259 F F F
183192023-12-10T18:57:06.847Z 16 320-339 149 149 149 260 260 260 F F F
183202023-12-10T18:57:06.847Z 17 340-359 148 148 148 257 257 257 F F F
183212023-12-10T18:57:06.847Z 18 360-379 148 148 148 259 259 259 F F F
183222023-12-10T18:57:06.847Z 19 380-399 148 148 148 259 259 259 F F F
183232023-12-10T18:57:06.847Z 20 400-419 143 143 143 249 249 249 F F F
183242023-12-10T18:57:06.848Z 21 420-439 138 138 138 238 238 238 F F F
183252023-12-10T18:57:06.848Z 22 440-459 142 142 142 245 245 245 F F F
183262023-12-10T18:57:06.848Z 23 460-479 146 146 146 253 253 253 F F F
183272023-12-10T18:57:06.848Z 24 480-499 142 142 142 245 245 245 F F F
183282023-12-10T18:57:06.848Z 25 500-519 140 140 140 240 240 240 F F F
183292023-12-10T18:57:06.848Z 26 520-539 148 148 148 259 259 259 F F F
183302023-12-10T18:57:06.849Z 27 540-559 148 148 148 259 259 259 F F F
183312023-12-10T18:57:06.849Z 28 560-579 146 146 146 255 255 255 F F F
183322023-12-10T18:57:06.849Z 29 580-599 146 146 146 253 253 253 F F F
183332023-12-10T18:57:06.849ZMax gen: 149, Max flush: 260
183342023-12-10T18:57:06.852Z
183352023-12-10T18:57:06.852ZBegin loop 75
183362023-12-10T18:57:06.852ZDownstairs to restart: 2
183372023-12-10T18:57:06.879Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.880918643Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762}
183382023-12-10T18:57:06.879Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.881340275Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762}
183392023-12-10T18:57:06.879Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.88136898Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762}
183402023-12-10T18:57:06.884Z{"msg":"Crucible e525dfe9-04eb-4a7f-8d5d-739ebeb43d48 has session id: 52720627-d398-4bb7-be36-c99bddf1ae8c","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.881417246Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183412023-12-10T18:57:06.884Z{"msg":"Upstairs opts: Upstairs UUID: e525dfe9-04eb-4a7f-8d5d-739ebeb43d48, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.881487433Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183422023-12-10T18:57:06.884Z{"msg":"Crucible stats registered with UUID: e525dfe9-04eb-4a7f-8d5d-739ebeb43d48","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.881536222Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183432023-12-10T18:57:06.884ZCrucible runtime is spawned
183442023-12-10T18:57:06.884Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.881708998Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c","task":"up_listen"}
183452023-12-10T18:57:06.884Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30The guest has requested activation with gen:150
183462023-12-10T18:57:06.884Z,"time":"2023-12-10T18:57:06.881796492Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183472023-12-10T18:57:06.884Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.881827384Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183482023-12-10T18:57:06.884Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.881847987Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"looper":"0","session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183492023-12-10T18:57:06.884Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.882196663Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"looper":"1","session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183502023-12-10T18:57:06.884Z{"msg":"[0] e525dfe9-04eb-4a7f-8d5d-739ebeb43d48 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.882315404Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"looper":"0","session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183512023-12-10T18:57:06.885Z{"msg":"[1] e525dfe9-04eb-4a7f-8d5d-739ebeb43d48 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.882396814Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"looper":"1","session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183522023-12-10T18:57:06.885Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.882426211Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"looper":"2","session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183532023-12-10T18:57:06.886Z{"msg":"[2] e525dfe9-04eb-4a7f-8d5d-739ebeb43d48 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.882521583Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"looper":"2","session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183542023-12-10T18:57:06.886Z{"msg":"e525dfe9-04eb-4a7f-8d5d-739ebeb43d48 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.882763028Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183552023-12-10T18:57:06.886Z{"msg":"Set desired generation to :150","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.882833872Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183562023-12-10T18:57:06.886Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.882966081Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183572023-12-10T18:57:06.886Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.883112814Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183582023-12-10T18:57:06.886Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.883367768Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183592023-12-10T18:57:06.886Z{"msg":"[1] e525dfe9-04eb-4a7f-8d5d-739ebeb43d48 (52720627-d398-4bb7-be36-c99bddf1ae8c) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.883728368Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183602023-12-10T18:57:06.886Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.883781251Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183612023-12-10T18:57:06.887Z{"msg":"[1] client is_active_req TRUE, promote! session 52720627-d398-4bb7-be36-c99bddf1ae8c","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.883806518Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183622023-12-10T18:57:06.887Z{"msg":"[0] e525dfe9-04eb-4a7f-8d5d-739ebeb43d48 (52720627-d398-4bb7-be36-c99bddf1ae8c) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.883918885Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183632023-12-10T18:57:06.893Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.883950845Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183642023-12-10T18:57:06.893Z{"msg":"[0] client is_active_req TRUE, promote! session 52720627-d398-4bb7-be36-c99bddf1ae8c","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.883975307Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183652023-12-10T18:57:06.893Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.884593365Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183662023-12-10T18:57:06.893Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.884684127Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183672023-12-10T18:57:06.893Z{"msg":"e525dfe9-04eb-4a7f-8d5d-739ebeb43d48 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.884793643Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183682023-12-10T18:57:06.893Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.88485583Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183692023-12-10T18:57:06.893Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.884884653Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183702023-12-10T18:57:06.905Z{"msg":"e525dfe9-04eb-4a7f-8d5d-739ebeb43d48 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.885064567Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183712023-12-10T18:57:06.906Z{"msg":"[2] e525dfe9-04eb-4a7f-8d5d-739ebeb43d48 (52720627-d398-4bb7-be36-c99bddf1ae8c) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.885174451Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183722023-12-10T18:57:06.906Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.885412422Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183732023-12-10T18:57:06.906Z{"msg":"[2] client is_active_req TRUE, promote! session 52720627-d398-4bb7-be36-c99bddf1ae8c","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.885462406Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183742023-12-10T18:57:06.906Z{"msg":"[1] e525dfe9-04eb-4a7f-8d5d-739ebeb43d48 (52720627-d398-4bb7-be36-c99bddf1ae8c) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.885546763Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183752023-12-10T18:57:06.906Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.885581443Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183762023-12-10T18:57:06.906Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:06.885640073Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183772023-12-10T18:57:06.906Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.885666093Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183782023-12-10T18:57:06.906Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.885724723Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183792023-12-10T18:57:06.906Z{"msg":"e525dfe9-04eb-4a7f-8d5d-739ebeb43d48 WaitActive WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.885756305Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183802023-12-10T18:57:06.906Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.885812928Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183812023-12-10T18:57:06.907Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.885837682Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183822023-12-10T18:57:06.907Z{"msg":"[0] e525dfe9-04eb-4a7f-8d5d-739ebeb43d48 (52720627-d398-4bb7-be36-c99bddf1ae8c) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.885866411Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183832023-12-10T18:57:06.907Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.885922329Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183842023-12-10T18:57:06.907Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:06.885946743Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183852023-12-10T18:57:06.907Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.88597402Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183862023-12-10T18:57:06.907Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.886029361Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183872023-12-10T18:57:06.907Z{"msg":"e525dfe9-04eb-4a7f-8d5d-739ebeb43d48 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.886054796Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183882023-12-10T18:57:06.907Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.886080403Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183892023-12-10T18:57:06.907Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.886135914Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183902023-12-10T18:57:06.907Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.886161718Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183912023-12-10T18:57:06.907Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.886193555Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183922023-12-10T18:57:06.908Z{"msg":"e525dfe9-04eb-4a7f-8d5d-739ebeb43d48 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.88624942Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183932023-12-10T18:57:06.908Z{"msg":"[2] e525dfe9-04eb-4a7f-8d5d-739ebeb43d48 (52720627-d398-4bb7-be36-c99bddf1ae8c) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.886666802Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183942023-12-10T18:57:06.908Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.886742905Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183952023-12-10T18:57:06.908Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:06.886785561Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183962023-12-10T18:57:06.908Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.886820858Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183972023-12-10T18:57:06.908Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.886885002Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183982023-12-10T18:57:06.908Z{"msg":"e525dfe9-04eb-4a7f-8d5d-739ebeb43d48 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.886913723Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
183992023-12-10T18:57:06.908Z{"msg":"[0]R flush_numbers[0..12]: [256, 253, 260, 260, 259, 257, 235, 257, 250, 245, 252, 251]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.886965775Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184002023-12-10T18:57:06.908Z{"msg":"[0]R generation[0..12]: [147, 146, 149, 149, 148, 148, 136, 148, 144, 142, 145, 144]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.886990281Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184012023-12-10T18:57:06.908Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:150
184022023-12-10T18:57:06.909ZWait for a query_work_queue command to finish before sending IO
184032023-12-10T18:57:06.910Z,"time":"2023-12-10T18:57:06.887019887Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184042023-12-10T18:57:06.910Z{"msg":"[1]R flush_numbers[0..12]: [256, 253, 260, 260, 259, 257, 235, 257, 250, 245, 252, 251]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.887174671Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184052023-12-10T18:57:06.910Z{"msg":"[1]R generation[0..12]: [147, 146, 149, 149, 148, 148, 136, 148, 144, 142, 145, 144]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.887248144Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184062023-12-10T18:57:06.910Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.887312522Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184072023-12-10T18:57:06.910Z{"msg":"[2]R flush_numbers[0..12]: [256, 253, 260, 260, 259, 257, 235, 257, 250, 245, 252, 251]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.887348406Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184082023-12-10T18:57:06.910Z{"msg":"[2]R generation[0..12]: [147, 146, 149, 149, 148, 148, 136, 148, 144, 142, 145, 144]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.887431321Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184092023-12-10T18:57:06.910Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.887506883Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184102023-12-10T18:57:06.910Z{"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.887532136Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184112023-12-10T18:57:06.910Z{"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.887553012Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184122023-12-10T18:57:06.910Z{"msg":"Next flush: 261","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.887578655Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184132023-12-10T18:57:06.911Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.887644976Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184142023-12-10T18:57:06.911Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.887688199Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184152023-12-10T18:57:06.912Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.887756046Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184162023-12-10T18:57:06.912Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.887789745Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184172023-12-10T18:57:06.912Z{"msg":"e525dfe9-04eb-4a7f-8d5d-739ebeb43d48 is now active with session: 52720627-d398-4bb7-be36-c99bddf1ae8c","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.88786336Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184182023-12-10T18:57:06.912Z{"msg":"e525dfe9-04eb-4a7f-8d5d-739ebeb43d48 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.887903297Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184192023-12-10T18:57:06.912Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.887949575Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184202023-12-10T18:57:06.912Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.887972416Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184212023-12-10T18:57:06.912Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.888001993Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184222023-12-10T18:57:06.912Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.888069143Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184232023-12-10T18:57:06.912Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.888096177Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184242023-12-10T18:57:06.913Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.888118374Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184252023-12-10T18:57:06.913Z{"msg":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
184262023-12-10T18:57:06.913Z"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.888231593Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184272023-12-10T18:57:06.913Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.888308166Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184282023-12-10T18:57:06.913Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.888378803Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184292023-12-10T18:57:06.913Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.888450258Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184302023-12-10T18:57:06.913Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:06.888516218Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1762,"session_id":"52720627-d398-4bb7-be36-c99bddf1ae8c"}
184312023-12-10T18:57:06.913ZLoading write count information from file "/tmp/test_repair_verify.data"
184322023-12-10T18:57:06.913ZRun Repair workload
184332023-12-10T18:57:06.913Z01/30 Write block 31 len 32768 data: 11 14 9 11 11 16 14 13
184342023-12-10T18:57:06.913Z02/30 Write block 48 len 12288 data: 9 9 8
184352023-12-10T18:57:06.913Z03/30 Read block 120 len 20480
184362023-12-10T18:57:06.913Z04/30 Read block 137 len 36864
184372023-12-10T18:57:06.913Z05/30 Read block 324 len 40960
184382023-12-10T18:57:06.917Z06/30 Write block 178 len 28672 data: 8 7 6 8 11 11 8
184392023-12-10T18:57:06.917Z07/30 Write block 412 len 4096 data: 7
184402023-12-10T18:57:06.917Z08/30 Read block 322 len 24576
184412023-12-10T18:57:06.917Z09/30 Read block 328 len 40960
184422023-12-10T18:57:06.917Z10/30 Flush
184432023-12-10T18:57:06.917Z11/30 Write block 65 len 40960 data: 15 16 13 15 16 14 14 11 14 15
184442023-12-10T18:57:06.917Z12/30 Read block 525 len 12288
184452023-12-10T18:57:06.917Z13/30 Write block 242 len 4096 data: 14
184462023-12-10T18:57:06.918Z14/30 Read block 270 len 20480
184472023-12-10T18:57:06.918Z15/30 Write block 531 len 16384 data: 16 15 14 16
184482023-12-10T18:57:06.918Z16/30 Write block 239 len 40960 data: 16 15 15 15 14 15 14 15 13 14
184492023-12-10T18:57:06.920Z17/30 Read block 590 len 40960
184502023-12-10T18:57:06.923Z18/30 Read block 494 len 16384
184512023-12-10T18:57:06.923Z19/30 Read block 535 len 4096
184522023-12-10T18:57:06.926Z20/30 Read block 433 len 40960
184532023-12-10T18:57:06.929Z21/30 Write block 20 len 32768 data: 12 10 11 10 11 10 10 7
184542023-12-10T18:57:06.929Z22/30 Read block 563 len 28672
184552023-12-10T18:57:06.933Z23/30 Read block 427 len 4096
184562023-12-10T18:57:06.933Z24/30 Read block 341 len 8192
184572023-12-10T18:57:06.936Z25/30 Read block 124 len 8192
184582023-12-10T18:57:06.936Z26/30 Write block 387 len 36864 data: 13 12 10 12 10 11 9 8 9
184592023-12-10T18:57:06.939Z27/30 Write block 284 len 36864 data: 12 14 15 15 13 13 10 11 12
184602023-12-10T18:57:06.939Z28/30 Write block 448 len 36864 data: 10 10 10 12 12 12 10 10 10
184612023-12-10T18:57:06.942Z29/30 Write block 248 len 20480 data: 15 15 14 14 13
184622023-12-10T18:57:06.942Z30/30 Read block 369 len 28672
184632023-12-10T18:57:06.945Z----------------------------------------------------------------
184642023-12-10T18:57:06.946Z Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30
184652023-12-10T18:57:06.946ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
184662023-12-10T18:57:06.946Z 1 Acked 1000 Write 8 Done Done Sent false
184672023-12-10T18:57:06.946Z 2 Acked 1001 Write 3 Done Done Sent false
184682023-12-10T18:57:06.946Z 3 Acked 1002 Read 5 Done Done Sent false
184692023-12-10T18:57:06.946Z 4 Acked 1003 Read 9 Done Done Sent false
184702023-12-10T18:57:06.946Z 5 Acked 1004 Read 10 Done Done Sent false
184712023-12-10T18:57:06.946Z 6 Acked 1005 Write 7 Done Done Sent false
184722023-12-10T18:57:06.946Z 7 Acked 1006 Write 1 Done Done Sent false
184732023-12-10T18:57:06.946Z 8 Acked 1007 Read 6 Done Done Sent false
184742023-12-10T18:57:06.946Z 9 Acked 1008 Read 10 Done Done Sent false
184752023-12-10T18:57:06.946Z 10 Acked 1009 Flush 0 Done Done Sent false
184762023-12-10T18:57:06.946Z 11 Acked 1010 Write 10 Done Done Sent false
184772023-12-10T18:57:06.946Z 12 Acked 1011 Read 3 Done Done Sent false
184782023-12-10T18:57:06.946Z 13 Acked 1012 Write 1 Done Done Sent false
184792023-12-10T18:57:06.946Z 14 Acked 1013 Read 5 Done Done Sent false
184802023-12-10T18:57:06.946Z 15 Acked 1014 Write 4 Done Done Sent false
184812023-12-10T18:57:06.947Z 16 Acked 1015 Write 10 Done Done Sent false
184822023-12-10T18:57:06.947Z 17 Acked 1016 Read 10 Done Done Sent false
184832023-12-10T18:57:06.947Z 18 Acked 1017 Read 4 Done Done Sent false
184842023-12-10T18:57:06.947Z 19 Acked 1018 Read 1 Done Done Sent false
184852023-12-10T18:57:06.947Z 20 Acked 1019 Read 10 Done Done Sent false
184862023-12-10T18:57:06.947Z 21 Acked 1020 Write 8 Done Done Sent false
184872023-12-10T18:57:06.947Z 22 Acked 1021 Read 7 Done Done Sent false
184882023-12-10T18:57:06.947Z 23 Acked 1022 Read 1 Done Done Sent false
184892023-12-10T18:57:06.948Z 24 Acked 1023 Read 2 Done Done Sent false
184902023-12-10T18:57:06.948Z 25 Acked 1024 Read 2 Done Done Sent false
184912023-12-10T18:57:06.948Z 26 Acked 1025 Write 9 Done Done Sent false
184922023-12-10T18:57:06.948Z 27 Acked 1026 Write 9 Done Done Sent false
184932023-12-10T18:57:06.948Z 28 Acked 1027 Write 9 Done Done Sent false
184942023-12-10T18:57:06.948Z 29 Acked 1028 Write 5 Done Done Sent false
184952023-12-10T18:57:06.948Z 30 Acked 1029 Read 7 Done Done Sent false
184962023-12-10T18:57:06.948Z STATES DS:0 DS:1 DS:2 TOTAL
184972023-12-10T18:57:06.948Z New 0 0 0 0
184982023-12-10T18:57:06.948Z Sent 0 0 30 30
184992023-12-10T18:57:06.950Z Done 30 30 0 60
185002023-12-10T18:57:06.950Z Skipped 0 0 0 0
185012023-12-10T18:57:06.950Z Error 0 0 0 0
185022023-12-10T18:57:06.950ZLast Flush: 1009 1009 0
185032023-12-10T18:57:06.950ZDownstairs last five completed:
185042023-12-10T18:57:06.950ZUpstairs last five completed: 30 29 28 27 26 25
185052023-12-10T18:57:06.950ZWrote out file "/tmp/test_repair_verify.data"
185062023-12-10T18:57:06.954Z
185072023-12-10T18:57:06.959ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
185082023-12-10T18:57:07.031ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
185092023-12-10T18:57:07.031Z 0 000-019 147 147 147 256 256 256 F F F
185102023-12-10T18:57:07.039Z 1 020-039 150 150 146 261 261 253 T T F <---
185112023-12-10T18:57:07.039Z 2 040-059 150 150 149 261 261 260 F F F <---
185122023-12-10T18:57:07.039Z 3 060-079 149 149 149 260 260 260 T T F <---
185132023-12-10T18:57:07.039Z 4 080-099 148 148 148 259 259 259 F F F
185142023-12-10T18:57:07.039Z 5 100-119 148 148 148 257 257 257 F F F
185152023-12-10T18:57:07.039Z 6 120-139 136 136 136 235 235 235 F F F
185162023-12-10T18:57:07.039Z 7 140-159 148 148 148 257 257 257 F F F
185172023-12-10T18:57:07.039Z 8 160-179 150 150 144 261 261 250 F F F <---
185182023-12-10T18:57:07.039Z 9 180-199 150 150 142 261 261 245 F F F <---
185192023-12-10T18:57:07.039Z 10 200-219 145 145 145 252 252 252 F F F
185202023-12-10T18:57:07.039Z 11 220-239 144 144 144 251 251 251 T T F <---
185212023-12-10T18:57:07.039Z 12 240-259 143 143 143 249 249 249 T T F <---
185222023-12-10T18:57:07.039Z 13 260-279 140 140 140 243 243 243 F F F
185232023-12-10T18:57:07.039Z 14 280-299 146 146 146 253 253 253 T T F <---
185242023-12-10T18:57:07.040Z 15 300-319 148 148 148 259 259 259 F F F
185252023-12-10T18:57:07.040Z 16 320-339 149 149 149 260 260 260 F F F
185262023-12-10T18:57:07.040Z 17 340-359 148 148 148 257 257 257 F F F
185272023-12-10T18:57:07.040Z 18 360-379 148 148 148 259 259 259 F F F
185282023-12-10T18:57:07.040Z 19 380-399 148 148 148 259 259 259 T T F <---
185292023-12-10T18:57:07.040Z 20 400-419 150 150 143 261 261 249 F F F <---
185302023-12-10T18:57:07.040Z 21 420-439 138 138 138 238 238 238 F F F
185312023-12-10T18:57:07.040Z 22 440-459 142 142 142 245 245 245 T T F <---
185322023-12-10T18:57:07.040Z 23 460-479 146 146 146 253 253 253 F F F
185332023-12-10T18:57:07.040Z 24 480-499 142 142 142 245 245 245 F F F
185342023-12-10T18:57:07.040Z 25 500-519 140 140 140 240 240 240 F F F
185352023-12-10T18:57:07.040Z 26 520-539 148 148 148 259 259 259 T T F <---
185362023-12-10T18:57:07.040Z 27 540-559 148 148 148 259 259 259 F F F
185372023-12-10T18:57:07.040Z 28 560-579 146 146 146 255 255 255 F F F
185382023-12-10T18:57:07.040Z 29 580-599 146 146 146 253 253 253 F F F
185392023-12-10T18:57:07.040ZMax gen: 150, Max flush: 261
185402023-12-10T18:57:07.040ZError: Difference in extent metadata found!
185412023-12-10T18:57:07.041ZA Difference in extent metadata is expected here
185422023-12-10T18:57:07.041ZOn loop 75
185432023-12-10T18:57:07.041Z
185442023-12-10T18:57:07.041ZVerifying data now
185452023-12-10T18:57:12.195ZNo change after verify
185462023-12-10T18:57:12.195ZLoop: 75 Downstairs dump after verify (and repair):
185472023-12-10T18:57:12.237ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
185482023-12-10T18:57:12.238Z 0 000-019 147 147 147 256 256 256 F F F
185492023-12-10T18:57:12.241Z 1 020-039 151 151 151 262 262 262 F F F
185502023-12-10T18:57:12.241Z 2 040-059 150 150 150 261 261 261 F F F
185512023-12-10T18:57:12.241Z 3 060-079 151 151 151 262 262 262 F F F
185522023-12-10T18:57:12.241Z 4 080-099 148 148 148 259 259 259 F F F
185532023-12-10T18:57:12.241Z 5 100-119 148 148 148 257 257 257 F F F
185542023-12-10T18:57:12.241Z 6 120-139 136 136 136 235 235 235 F F F
185552023-12-10T18:57:12.242Z 7 140-159 148 148 148 257 257 257 F F F
185562023-12-10T18:57:12.242Z 8 160-179 150 150 150 261 261 261 F F F
185572023-12-10T18:57:12.242Z 9 180-199 150 150 150 261 261 261 F F F
185582023-12-10T18:57:12.242Z 10 200-219 145 145 145 252 252 252 F F F
185592023-12-10T18:57:12.242Z 11 220-239 151 151 151 262 262 262 F F F
185602023-12-10T18:57:12.242Z 12 240-259 151 151 151 262 262 262 F F F
185612023-12-10T18:57:12.242Z 13 260-279 140 140 140 243 243 243 F F F
185622023-12-10T18:57:12.242Z 14 280-299 151 151 151 262 262 262 F F F
185632023-12-10T18:57:12.242Z 15 300-319 148 148 148 259 259 259 F F F
185642023-12-10T18:57:12.242Z 16 320-339 149 149 149 260 260 260 F F F
185652023-12-10T18:57:12.242Z 17 340-359 148 148 148 257 257 257 F F F
185662023-12-10T18:57:12.242Z 18 360-379 148 148 148 259 259 259 F F F
185672023-12-10T18:57:12.242Z 19 380-399 151 151 151 262 262 262 F F F
185682023-12-10T18:57:12.242Z 20 400-419 150 150 150 261 261 261 F F F
185692023-12-10T18:57:12.242Z 21 420-439 138 138 138 238 238 238 F F F
185702023-12-10T18:57:12.242Z 22 440-459 151 151 151 262 262 262 F F F
185712023-12-10T18:57:12.242Z 23 460-479 146 146 146 253 253 253 F F F
185722023-12-10T18:57:12.242Z 24 480-499 142 142 142 245 245 245 F F F
185732023-12-10T18:57:12.243Z 25 500-519 140 140 140 240 240 240 F F F
185742023-12-10T18:57:12.243Z 26 520-539 151 151 151 262 262 262 F F F
185752023-12-10T18:57:12.243Z 27 540-559 148 148 148 259 259 259 F F F
185762023-12-10T18:57:12.243Z 28 560-579 146 146 146 255 255 255 F F F
185772023-12-10T18:57:12.243Z 29 580-599 146 146 146 253 253 253 F F F
185782023-12-10T18:57:12.243ZMax gen: 151, Max flush: 262
185792023-12-10T18:57:12.243Z
185802023-12-10T18:57:12.243ZBegin loop 76
185812023-12-10T18:57:12.243ZDownstairs to restart: 0
185822023-12-10T18:57:12.256Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.25622239Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770}
185832023-12-10T18:57:12.256Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.256699434Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770}
185842023-12-10T18:57:12.256Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.256750642Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770}
185852023-12-10T18:57:12.262Z{"msg":"Crucible 4f26215f-318e-4077-86e4-c6530f470f8e has session id: d7b08175-0686-490f-9006-76c5f88262a1","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.256780994Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
185862023-12-10T18:57:12.262Z{"msg":"Upstairs opts: Upstairs UUID: 4f26215f-318e-4077-86e4-c6530f470f8e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.256864846Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
185872023-12-10T18:57:12.262Z{"msg":"Crucible stats registered with UUID: 4f26215f-318e-4077-86e4-c6530f470f8e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.256914111Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":Crucible runtime is spawned
185882023-12-10T18:57:12.262Z1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
185892023-12-10T18:57:12.263ZThe guest has requested activation with gen:152
185902023-12-10T18:57:12.263Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.257088416Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"looper":"0","session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
185912023-12-10T18:57:12.263Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.257151891Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"looper":"2","session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
185922023-12-10T18:57:12.263Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.257193165Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"looper":"1","session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
185932023-12-10T18:57:12.263Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.257303907Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1","task":"up_listen"}
185942023-12-10T18:57:12.263Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.257347049Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
185952023-12-10T18:57:12.263Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.257413344Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
185962023-12-10T18:57:12.263Z{"msg":"4f26215f-318e-4077-86e4-c6530f470f8e active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.257501784Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
185972023-12-10T18:57:12.263Z{"msg":"Set desired generation to :152","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.257537792Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
185982023-12-10T18:57:12.263Z{"msg":"[0] 4f26215f-318e-4077-86e4-c6530f470f8e looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.257712235Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"looper":"0","session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
185992023-12-10T18:57:12.263Z{"msg":"[1] 4f26215f-318e-4077-86e4-c6530f470f8e looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.25779582Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"looper":"1","session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186002023-12-10T18:57:12.264Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.257841543Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186012023-12-10T18:57:12.264Z{"msg":"[2] 4f26215f-318e-4077-86e4-c6530f470f8e looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.257882393Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"looper":"2","session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186022023-12-10T18:57:12.264Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.257920791Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186032023-12-10T18:57:12.264Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.25799035Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186042023-12-10T18:57:12.264Z{"msg":"[1] 4f26215f-318e-4077-86e4-c6530f470f8e (d7b08175-0686-490f-9006-76c5f88262a1) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.258592571Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186052023-12-10T18:57:12.264Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.258668377Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186062023-12-10T18:57:12.265Z{"msg":"[2] 4f26215f-318e-4077-86e4-c6530f470f8e (d7b08175-0686-490f-9006-76c5f88262a1) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.258714115Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186072023-12-10T18:57:12.265Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.258750315Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186082023-12-10T18:57:12.265Z{"msg":"[1] client is_active_req TRUE, promote! session d7b08175-0686-490f-9006-76c5f88262a1","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.258849193Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186092023-12-10T18:57:12.265Z{"msg":"[2] client is_active_req TRUE, promote! session d7b08175-0686-490f-9006-76c5f88262a1","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.258889992Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186102023-12-10T18:57:12.265Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.259494222Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186112023-12-10T18:57:12.265Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.259559537Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186122023-12-10T18:57:12.265Z{"msg":"4f26215f-318e-4077-86e4-c6530f470f8e New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.259585373Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186132023-12-10T18:57:12.265Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.259687511Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186142023-12-10T18:57:12.265Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.25980712Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186152023-12-10T18:57:12.265Z{"msg":"4f26215f-318e-4077-86e4-c6530f470f8e New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.259881765Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186162023-12-10T18:57:12.265Z{"msg":"[0] 4f26215f-318e-4077-86e4-c6530f470f8e (d7b08175-0686-490f-9006-76c5f88262a1) New WaitActive WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.259911417Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186172023-12-10T18:57:12.265Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.259939721Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186182023-12-10T18:57:12.265Z{"msg":"[0] client is_active_req TRUE, promote! session d7b08175-0686-490f-9006-76c5f88262a1","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.259974758Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186192023-12-10T18:57:12.265Z{"msg":"[1] 4f26215f-318e-4077-86e4-c6530f470f8e (d7b08175-0686-490f-9006-76c5f88262a1) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.26033396Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186202023-12-10T18:57:12.265Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.260397108Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186212023-12-10T18:57:12.265Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:12.260422243Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186222023-12-10T18:57:12.265Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.260465431Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186232023-12-10T18:57:12.265Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.260489152Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186242023-12-10T18:57:12.266Z{"msg":"4f26215f-318e-4077-86e4-c6530f470f8e WaitActive WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.260510354Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186252023-12-10T18:57:12.266Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.260530648Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186262023-12-10T18:57:12.266Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.260635329Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186272023-12-10T18:57:12.266Z{"msg":"[2] 4f26215f-318e-4077-86e4-c6530f470f8e (d7b08175-0686-490f-9006-76c5f88262a1) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.260661801Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186282023-12-10T18:57:12.266Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.260712285Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186292023-12-10T18:57:12.267Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:12.260735062Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186302023-12-10T18:57:12.267Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.260754996Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186312023-12-10T18:57:12.267Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.260795243Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186322023-12-10T18:57:12.267Z{"msg":"4f26215f-318e-4077-86e4-c6530f470f8e WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.260818541Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186332023-12-10T18:57:12.267Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.260839195Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186342023-12-10T18:57:12.267Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.26087882Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186352023-12-10T18:57:12.267Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.260922417Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186362023-12-10T18:57:12.267Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.260980129Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186372023-12-10T18:57:12.267Z{"msg":"4f26215f-318e-4077-86e4-c6530f470f8e WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.261004339Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186382023-12-10T18:57:12.267Z{"msg":"[0] 4f26215f-318e-4077-86e4-c6530f470f8e (d7b08175-0686-490f-9006-76c5f88262a1) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.26158611Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186392023-12-10T18:57:12.268Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.261695321Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186402023-12-10T18:57:12.273Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:12.261726196Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186412023-12-10T18:57:12.273Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.26178729Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186422023-12-10T18:57:12.273Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.261810986Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186432023-12-10T18:57:12.273Z{"msg":"4f26215f-318e-4077-86e4-c6530f470f8e WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.261831881Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186442023-12-10T18:57:12.273Z{"msg":"[0]R flush_numbers[0..12]: [256, 262, 261, 262, 259, 257, 235, 257, 261, 261, 252, 262]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.261853596Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186452023-12-10T18:57:12.273Z{"msg":"[0]R generation[0..12]: [147, 151, 150, 151, 148, 148, 136, 148, 150, 150, 145, 151]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.261875635Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186462023-12-10T18:57:12.273Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.261905443Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186472023-12-10T18:57:12.273Z{"msg":"[1]R flush_numbers[0..12]: [256, 262, 261, 262, 259, 257, 235, 257, 261, 261, 252, 262]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.261927989Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186482023-12-10T18:57:12.273ZThe guest has finished waiting for activation with:152
186492023-12-10T18:57:12.273Z{"msg":"[1]R generation[0..12]: [147, 151, 150, 151, 148, 148, 136, 148, 150, 150, 145, 151]","v":0,"Wait for a query_work_queue command to finish before sending IO
186502023-12-10T18:57:12.276Zname":"crucible","level":30,"time":"2023-12-10T18:57:12.261958052Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186512023-12-10T18:57:12.276Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.261980372Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186522023-12-10T18:57:12.276Z{"msg":"[2]R flush_numbers[0..12]: [256, 262, 261, 262, 259, 257, 235, 257, 261, 261, 252, 262]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.26200207Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186532023-12-10T18:57:12.276Z{"msg":"[2]R generation[0..12]: [147, 151, 150, 151, 148, 148, 136, 148, 150, 150, 145, 151]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.262049887Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186542023-12-10T18:57:12.276Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.262074289Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186552023-12-10T18:57:12.276Z{"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.262094198Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186562023-12-10T18:57:12.276Z{"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.26217269Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186572023-12-10T18:57:12.276Z{"msg":"Next flush: 263","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.262214781Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186582023-12-10T18:57:12.276Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.262316598Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186592023-12-10T18:57:12.277Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.262408872Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186602023-12-10T18:57:12.277Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.262487528Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186612023-12-10T18:57:12.278Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.262520639Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186622023-12-10T18:57:12.278Z{"msg":"4f26215f-318e-4077-86e4-c6530f470f8e is now active with session: d7b08175-0686-490f-9006-76c5f88262a1","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.262544987Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186632023-12-10T18:57:12.278Z{"msg":"4f26215f-318e-4077-86e4-c6530f470f8e Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.262591052Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186642023-12-10T18:57:12.278Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.262614063Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186652023-12-10T18:57:12.278Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.262686153Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186662023-12-10T18:57:12.278Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.262715072Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186672023-12-10T18:57:12.278Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.262735632Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186682023-12-10T18:57:12.278Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.262789254Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186692023-12-10T18:57:12.278Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.262811445Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186702023-12-10T18:57:12.278Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.262831564Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186712023-12-10T18:57:12.278Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.262889028Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186722023-12-10T18:57:12.280Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.262915043Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186732023-12-10T18:57:12.280Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.262957519Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186742023-12-10T18:57:12.280Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:12.262981613Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1770,"session_id":"d7b08175-0686-490f-9006-76c5f88262a1"}
186752023-12-10T18:57:12.281ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
186762023-12-10T18:57:12.281ZLoading write count information from file "/tmp/test_repair_verify.data"
186772023-12-10T18:57:12.281ZRun Repair workload
186782023-12-10T18:57:12.281Z01/30 Write block 153 len 12288 data: 11 15 11
186792023-12-10T18:57:12.281Z02/30 Write block 461 len 4096 data: 9
186802023-12-10T18:57:12.281Z03/30 Read block 322 len 12288
186812023-12-10T18:57:12.281Z04/30 Write block 157 len 40960 data: 10 8 10 10 11 17 17 16 15 16
186822023-12-10T18:57:12.281Z05/30 Write block 115 len 16384 data: 13 14 13 13
186832023-12-10T18:57:12.281Z06/30 Write block 229 len 28672 data: 13 19 18 17 14 17 17
186842023-12-10T18:57:12.281Z07/30 Read block 36 len 8192
186852023-12-10T18:57:12.285Z08/30 Write block 380 len 36864 data: 19 14 14 11 13 14 14 14 13
186862023-12-10T18:57:12.285Z09/30 Read block 356 len 24576
186872023-12-10T18:57:12.285Z10/30 Flush
186882023-12-10T18:57:12.285Z11/30 Flush
186892023-12-10T18:57:12.286Z12/30 Write block 169 len 36864 data: 14 13 14 10 11 11 9 9 10
186902023-12-10T18:57:12.286Z13/30 Write block 446 len 24576 data: 9 9 11 11 11 13
186912023-12-10T18:57:12.286Z14/30 Read block 337 len 4096
186922023-12-10T18:57:12.286Z15/30 Write block 214 len 16384 data: 11 13 16 16
186932023-12-10T18:57:12.286Z16/30 Read block 318 len 4096
186942023-12-10T18:57:12.290Z17/30 Read block 439 len 8192
186952023-12-10T18:57:12.290Z18/30 Read block 572 len 40960
186962023-12-10T18:57:12.293Z19/30 Read block 594 len 24576
186972023-12-10T18:57:12.296Z20/30 Read block 170 len 32768
186982023-12-10T18:57:12.299Z21/30 Write block 430 len 40960 data: 10 9 8 8 6 6 4 5 7 7
186992023-12-10T18:57:12.299Z22/30 Read block 523 len 40960
187002023-12-10T18:57:12.302Z23/30 Flush
187012023-12-10T18:57:12.309Z24/30 Write block 422 len 24576 data: 13 13 15 12 12 13
187022023-12-10T18:57:12.309Z25/30 Write block 255 len 12288 data: 10 10 8
187032023-12-10T18:57:12.312Z26/30 Read block 337 len 8192
187042023-12-10T18:57:12.312Z27/30 Write block 562 len 8192 data: 15 12
187052023-12-10T18:57:12.312Z28/30 Write block 291 len 28672 data: 12 13 12 9 8 11 11
187062023-12-10T18:57:12.312Z29/30 Write block 44 len 16384 data: 8 11 9 9
187072023-12-10T18:57:12.312Z30/30 Read block 234 len 16384
187082023-12-10T18:57:12.316Z----------------------------------------------------------------
187092023-12-10T18:57:12.316Z Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30
187102023-12-10T18:57:12.316ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
187112023-12-10T18:57:12.316Z 1 Acked 1000 Write 3 Done Done Done false
187122023-12-10T18:57:12.317Z 2 Acked 1001 Write 1 Done Done Done false
187132023-12-10T18:57:12.317Z 3 Acked 1002 Read 3 Done Done Done false
187142023-12-10T18:57:12.317Z 4 Acked 1003 Write 10 Done Done Done false
187152023-12-10T18:57:12.317Z 5 Acked 1004 Write 4 Done Done Done false
187162023-12-10T18:57:12.317Z 6 Acked 1005 Write 7 Sent Done Done false
187172023-12-10T18:57:12.317Z 7 Acked 1006 Read 2 Sent Done Done false
187182023-12-10T18:57:12.318Z 8 Acked 1007 Write 9 Sent Done Done false
187192023-12-10T18:57:12.318Z 9 Acked 1008 Read 6 Sent Done Done false
187202023-12-10T18:57:12.318Z 10 Acked 1009 Flush 0 Sent Done Done false
187212023-12-10T18:57:12.318Z 11 Acked 1010 Flush 0 Sent Done Done false
187222023-12-10T18:57:12.318Z 12 Acked 1011 Write 9 Sent Done Done false
187232023-12-10T18:57:12.318Z 13 Acked 1012 Write 6 Sent Done Done false
187242023-12-10T18:57:12.318Z 14 Acked 1013 Read 1 Sent Done Done false
187252023-12-10T18:57:12.319Z 15 Acked 1014 Write 4 Sent Done Done false
187262023-12-10T18:57:12.319Z 16 Acked 1015 Read 1 Sent Done Done false
187272023-12-10T18:57:12.319Z 17 Acked 1016 Read 2 Sent Done Done false
187282023-12-10T18:57:12.319Z 18 Acked 1017 Read 10 Sent Done Done false
187292023-12-10T18:57:12.319Z 19 Acked 1018 Read 6 Sent Done Done false
187302023-12-10T18:57:12.319Z 20 Acked 1019 Read 8 Sent Done Done false
187312023-12-10T18:57:12.319Z 21 Acked 1020 Write 10 Sent Done Done false
187322023-12-10T18:57:12.319Z 22 Acked 1021 Read 10 Sent Done Done false
187332023-12-10T18:57:12.319Z 23 Acked 1022 Flush 0 Sent Done Done false
187342023-12-10T18:57:12.319Z 24 Acked 1023 Write 6 Sent Done Done false
187352023-12-10T18:57:12.319Z 25 Acked 1024 Write 3 Sent Done Done false
187362023-12-10T18:57:12.320Z 26 Acked 1025 Read 2 Sent Done Done false
187372023-12-10T18:57:12.320Z 27 Acked 1026 Write 2 Sent Done Done false
187382023-12-10T18:57:12.320Z 28 Acked 1027 Write 7 Sent Done Done false
187392023-12-10T18:57:12.320Z 29 Acked 1028 Write 4 Sent Done Done false
187402023-12-10T18:57:12.320Z 30 Acked 1029 Read 4 Sent Sent Done false
187412023-12-10T18:57:12.322Z STATES DS:0 DS:1 DS:2 TOTAL
187422023-12-10T18:57:12.322Z New 0 0 0 0
187432023-12-10T18:57:12.322Z Sent 25 1 0 26
187442023-12-10T18:57:12.322Z Done 5 29 30 64
187452023-12-10T18:57:12.322Z Skipped 0 0 0 0
187462023-12-10T18:57:12.322Z Error 0 0 0 0
187472023-12-10T18:57:12.322ZLast Flush: 0 1022 1022
187482023-12-10T18:57:12.322ZDownstairs last five completed:
187492023-12-10T18:57:12.322ZUpstairs last five completed: 30 29 28 27 26 25
187502023-12-10T18:57:12.323ZWrote out file "/tmp/test_repair_verify.data"
187512023-12-10T18:57:12.326Z
187522023-12-10T18:57:12.326ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
187532023-12-10T18:57:12.353ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
187542023-12-10T18:57:12.353Z 0 000-019 147 147 147 256 256 256 F F F
187552023-12-10T18:57:12.356Z 1 020-039 151 151 151 262 262 262 F F F
187562023-12-10T18:57:12.356Z 2 040-059 150 150 150 261 261 261 F T T <---
187572023-12-10T18:57:12.356Z 3 060-079 151 151 151 262 262 262 F F F
187582023-12-10T18:57:12.356Z 4 080-099 148 148 148 259 259 259 F F F
187592023-12-10T18:57:12.357Z 5 100-119 148 152 152 257 263 263 T F F <---
187602023-12-10T18:57:12.357Z 6 120-139 136 136 136 235 235 235 F F F
187612023-12-10T18:57:12.357Z 7 140-159 148 152 152 257 263 263 T F F <---
187622023-12-10T18:57:12.357Z 8 160-179 150 152 152 261 265 265 T F F <---
187632023-12-10T18:57:12.357Z 9 180-199 150 150 150 261 261 261 F F F
187642023-12-10T18:57:12.357Z 10 200-219 145 152 152 252 265 265 F F F <---
187652023-12-10T18:57:12.357Z 11 220-239 151 152 152 262 263 263 F F F <---
187662023-12-10T18:57:12.357Z 12 240-259 151 151 151 262 262 262 F T T <---
187672023-12-10T18:57:12.357Z 13 260-279 140 140 140 243 243 243 F F F
187682023-12-10T18:57:12.357Z 14 280-299 151 151 151 262 262 262 F T T <---
187692023-12-10T18:57:12.357Z 15 300-319 148 148 148 259 259 259 F F F
187702023-12-10T18:57:12.357Z 16 320-339 149 149 149 260 260 260 F F F
187712023-12-10T18:57:12.357Z 17 340-359 148 148 148 257 257 257 F F F
187722023-12-10T18:57:12.358Z 18 360-379 148 148 148 259 259 259 F F F
187732023-12-10T18:57:12.358Z 19 380-399 151 152 152 262 263 263 F F F <---
187742023-12-10T18:57:12.358Z 20 400-419 150 150 150 261 261 261 F F F
187752023-12-10T18:57:12.358Z 21 420-439 138 152 152 238 265 265 F T T <---
187762023-12-10T18:57:12.358Z 22 440-459 151 152 152 262 265 265 F F F <---
187772023-12-10T18:57:12.358Z 23 460-479 146 152 152 253 263 263 T F F <---
187782023-12-10T18:57:12.358Z 24 480-499 142 142 142 245 245 245 F F F
187792023-12-10T18:57:12.358Z 25 500-519 140 140 140 240 240 240 F F F
187802023-12-10T18:57:12.358Z 26 520-539 151 151 151 262 262 262 F F F
187812023-12-10T18:57:12.358Z 27 540-559 148 148 148 259 259 259 F F F
187822023-12-10T18:57:12.358Z 28 560-579 146 146 146 255 255 255 F T T <---
187832023-12-10T18:57:12.358Z 29 580-599 146 146 146 253 253 253 F F F
187842023-12-10T18:57:12.358ZMax gen: 152, Max flush: 265
187852023-12-10T18:57:12.358ZError: Difference in extent metadata found!
187862023-12-10T18:57:12.367ZA Difference in extent metadata is expected here
187872023-12-10T18:57:12.367ZOn loop 76
187882023-12-10T18:57:12.367Z
187892023-12-10T18:57:12.367ZVerifying data now
187902023-12-10T18:57:18.635ZNo change after verify
187912023-12-10T18:57:18.635ZLoop: 76 Downstairs dump after verify (and repair):
187922023-12-10T18:57:18.658ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
187932023-12-10T18:57:18.658Z 0 000-019 147 147 147 256 256 256 F F F
187942023-12-10T18:57:18.658Z 1 020-039 151 151 151 262 262 262 F F F
187952023-12-10T18:57:18.658Z 2 040-059 153 153 153 266 266 266 F F F
187962023-12-10T18:57:18.658Z 3 060-079 151 151 151 262 262 262 F F F
187972023-12-10T18:57:18.658Z 4 080-099 148 148 148 259 259 259 F F F
187982023-12-10T18:57:18.661Z 5 100-119 152 152 152 263 263 263 F F F
187992023-12-10T18:57:18.661Z 6 120-139 136 136 136 235 235 235 F F F
188002023-12-10T18:57:18.661Z 7 140-159 152 152 152 263 263 263 F F F
188012023-12-10T18:57:18.661Z 8 160-179 152 152 152 265 265 265 F F F
188022023-12-10T18:57:18.661Z 9 180-199 150 150 150 261 261 261 F F F
188032023-12-10T18:57:18.662Z 10 200-219 152 152 152 265 265 265 F F F
188042023-12-10T18:57:18.662Z 11 220-239 152 152 152 263 263 263 F F F
188052023-12-10T18:57:18.662Z 12 240-259 153 153 153 266 266 266 F F F
188062023-12-10T18:57:18.662Z 13 260-279 140 140 140 243 243 243 F F F
188072023-12-10T18:57:18.663Z 14 280-299 153 153 153 266 266 266 F F F
188082023-12-10T18:57:18.663Z 15 300-319 148 148 148 259 259 259 F F F
188092023-12-10T18:57:18.663Z 16 320-339 149 149 149 260 260 260 F F F
188102023-12-10T18:57:18.663Z 17 340-359 148 148 148 257 257 257 F F F
188112023-12-10T18:57:18.663Z 18 360-379 148 148 148 259 259 259 F F F
188122023-12-10T18:57:18.663Z 19 380-399 152 152 152 263 263 263 F F F
188132023-12-10T18:57:18.663Z 20 400-419 150 150 150 261 261 261 F F F
188142023-12-10T18:57:18.663Z 21 420-439 153 153 153 266 266 266 F F F
188152023-12-10T18:57:18.663Z 22 440-459 152 152 152 265 265 265 F F F
188162023-12-10T18:57:18.663Z 23 460-479 152 152 152 263 263 263 F F F
188172023-12-10T18:57:18.663Z 24 480-499 142 142 142 245 245 245 F F F
188182023-12-10T18:57:18.663Z 25 500-519 140 140 140 240 240 240 F F F
188192023-12-10T18:57:18.663Z 26 520-539 151 151 151 262 262 262 F F F
188202023-12-10T18:57:18.663Z 27 540-559 148 148 148 259 259 259 F F F
188212023-12-10T18:57:18.663Z 28 560-579 153 153 153 266 266 266 F F F
188222023-12-10T18:57:18.664Z 29 580-599 146 146 146 253 253 253 F F F
188232023-12-10T18:57:18.664ZMax gen: 153, Max flush: 266
188242023-12-10T18:57:18.667Z
188252023-12-10T18:57:18.667ZBegin loop 77
188262023-12-10T18:57:18.667ZDownstairs to restart: 2
188272023-12-10T18:57:18.695Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.696286589Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778}
188282023-12-10T18:57:18.700Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.696748666Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778}
188292023-12-10T18:57:18.700Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.696875225Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778}
188302023-12-10T18:57:18.700Z{"msg":"Crucible cb8456ab-72ee-4ff3-9ae4-098f41b719e3 has session id: 9ac23147-2505-45c0-ab27-a02df26dd036","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.696938279Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188312023-12-10T18:57:18.700Z{"msg":"Crucible runtime is spawned
188322023-12-10T18:57:18.700ZUpstairs opts: Upstairs UUID: cb8456ab-72ee-4ff3-9ae4-098f41b719e3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.69703649Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188332023-12-10T18:57:18.700Z{"msg":"Crucible stats registered with UUID: cb8456ab-72ee-4ff3-9ae4-098f41b719e3","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.697082819Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188342023-12-10T18:57:18.700Z{"msg":"The guest has requested activation with gen:154
188352023-12-10T18:57:18.700Zup_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.697255078Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036","task":"up_listen"}
188362023-12-10T18:57:18.700Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.697370516Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188372023-12-10T18:57:18.700Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.697473814Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188382023-12-10T18:57:18.700Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.697512876Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"looper":"2","session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188392023-12-10T18:57:18.702Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.697561235Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"looper":"1","session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188402023-12-10T18:57:18.702Z{"msg":"[2] cb8456ab-72ee-4ff3-9ae4-098f41b719e3 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.697623578Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"looper":"2","session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188412023-12-10T18:57:18.702Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.697665736Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"looper":"0","session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188422023-12-10T18:57:18.702Z{"msg":"[1] cb8456ab-72ee-4ff3-9ae4-098f41b719e3 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.697733654Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"looper":"1","session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188432023-12-10T18:57:18.702Z{"msg":"[0] cb8456ab-72ee-4ff3-9ae4-098f41b719e3 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.697811144Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"looper":"0","session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188442023-12-10T18:57:18.702Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.698137592Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188452023-12-10T18:57:18.702Z{"msg":"cb8456ab-72ee-4ff3-9ae4-098f41b719e3 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.6981992Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188462023-12-10T18:57:18.702Z{"msg":"Set desired generation to :154","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.69822419Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188472023-12-10T18:57:18.702Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.698482597Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188482023-12-10T18:57:18.702Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.698535993Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188492023-12-10T18:57:18.702Z{"msg":"[1] cb8456ab-72ee-4ff3-9ae4-098f41b719e3 (9ac23147-2505-45c0-ab27-a02df26dd036) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.699007887Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188502023-12-10T18:57:18.703Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.699073285Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188512023-12-10T18:57:18.703Z{"msg":"[1] client is_active_req TRUE, promote! session 9ac23147-2505-45c0-ab27-a02df26dd036","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.699098486Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188522023-12-10T18:57:18.703Z{"msg":"[0] cb8456ab-72ee-4ff3-9ae4-098f41b719e3 (9ac23147-2505-45c0-ab27-a02df26dd036) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.699289274Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188532023-12-10T18:57:18.703Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.699345279Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188542023-12-10T18:57:18.703Z{"msg":"[0] client is_active_req TRUE, promote! session 9ac23147-2505-45c0-ab27-a02df26dd036","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.699386724Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188552023-12-10T18:57:18.710Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.699947126Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188562023-12-10T18:57:18.710Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.700061873Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188572023-12-10T18:57:18.710Z{"msg":"cb8456ab-72ee-4ff3-9ae4-098f41b719e3 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.700107662Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188582023-12-10T18:57:18.710Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.700169931Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188592023-12-10T18:57:18.711Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.700218508Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188602023-12-10T18:57:18.711Z{"msg":"cb8456ab-72ee-4ff3-9ae4-098f41b719e3 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.700276267Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188612023-12-10T18:57:18.711Z{"msg":"[1] cb8456ab-72ee-4ff3-9ae4-098f41b719e3 (9ac23147-2505-45c0-ab27-a02df26dd036) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.700763327Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188622023-12-10T18:57:18.711Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.700830438Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188632023-12-10T18:57:18.711Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:18.700870228Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188642023-12-10T18:57:18.711Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.700904504Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188652023-12-10T18:57:18.711Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.700963668Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188662023-12-10T18:57:18.711Z{"msg":"cb8456ab-72ee-4ff3-9ae4-098f41b719e3 WaitActive WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.700989689Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188672023-12-10T18:57:18.711Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.701014808Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188682023-12-10T18:57:18.711Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.701068234Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188692023-12-10T18:57:18.712Z{"msg":"[0] cb8456ab-72ee-4ff3-9ae4-098f41b719e3 (9ac23147-2505-45c0-ab27-a02df26dd036) WaitActive WaitQuorum New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.701236098Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188702023-12-10T18:57:18.712Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.701296015Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188712023-12-10T18:57:18.713Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:18.701320362Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188722023-12-10T18:57:18.713Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.701341471Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188732023-12-10T18:57:18.713Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.701387757Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188742023-12-10T18:57:18.713Z{"msg":"cb8456ab-72ee-4ff3-9ae4-098f41b719e3 WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.701414364Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188752023-12-10T18:57:18.714Z{"msg":"[2] cb8456ab-72ee-4ff3-9ae4-098f41b719e3 (9ac23147-2505-45c0-ab27-a02df26dd036) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.701461669Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188762023-12-10T18:57:18.714Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.701484594Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188772023-12-10T18:57:18.714Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.701504841Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188782023-12-10T18:57:18.714Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.70152511Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188792023-12-10T18:57:18.714Z{"msg":"[2] client is_active_req TRUE, promote! session 9ac23147-2505-45c0-ab27-a02df26dd036","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.701587579Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188802023-12-10T18:57:18.714Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.702307952Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188812023-12-10T18:57:18.714Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.702370059Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188822023-12-10T18:57:18.715Z{"msg":"cb8456ab-72ee-4ff3-9ae4-098f41b719e3 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.702395713Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188832023-12-10T18:57:18.715Z{"msg":"[2] cb8456ab-72ee-4ff3-9ae4-098f41b719e3 (9ac23147-2505-45c0-ab27-a02df26dd036) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.702956664Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188842023-12-10T18:57:18.715Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.703014925Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188852023-12-10T18:57:18.715Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:18.703038445Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188862023-12-10T18:57:18.715Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.703058418Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188872023-12-10T18:57:18.715Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.703078962Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188882023-12-10T18:57:18.715Z{"msg":"cb8456ab-72ee-4ff3-9ae4-098f41b719e3 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.703119825Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188892023-12-10T18:57:18.715Z{"msg":"[0]R flush_numbers[0..12]: [256, 262, 266, 262, 259, 263, 235, 263, 265, 261, 265, 263]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.703144077Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188902023-12-10T18:57:18.716Z{"msg":"[0]R generation[0..12]: [147, 151, 153, 151, 148, 152, 136, 152, 152, 150, 152, 152]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.703165912Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188912023-12-10T18:57:18.716Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.703207812Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188922023-12-10T18:57:18.716Z{"msg":"[1]R flush_numbers[0..12]: [256, 262, 266, 262, 259, 263, 235, 263, 265, 261, 265, 263]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.703231497Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188932023-12-10T18:57:18.716Z{"msg":"[1]R generation[0..12]: [147, 151, 153, 151, 148, 152, 136, 152, 152, 150, 152, 152]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.703252863Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188942023-12-10T18:57:18.716Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.703295618Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188952023-12-10T18:57:18.716Z{"msg":"[2]R flush_numbers[0..12]: [256, 262, 266, 262, 259, 263, 235, 263, 265, 261, 265, 263]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.703319456Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188962023-12-10T18:57:18.716Z{"msg":"[2]R generation[0..12]: [147, 151, 153, 151, 148, 152, 136, 152, 152, 150, 152, 152]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.70337541Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
188972023-12-10T18:57:18.716ZThe guest has finished waiting for activation with:154
188982023-12-10T18:57:18.716Z{"msg":"Wait for a query_work_queue command to finish before sending IO
188992023-12-10T18:57:18.716Z[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.703404506Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
189002023-12-10T18:57:18.717Z{"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.703424772Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
189012023-12-10T18:57:18.717Z{"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.703479008Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
189022023-12-10T18:57:18.717Z{"msg":"Next flush: 267","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.703529201Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
189032023-12-10T18:57:18.717Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.70355948Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
189042023-12-10T18:57:18.718Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.703584379Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
189052023-12-10T18:57:18.718Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.703635861Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
189062023-12-10T18:57:18.718Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.70365861Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
189072023-12-10T18:57:18.718Z{"msg":"cb8456ab-72ee-4ff3-9ae4-098f41b719e3 is now active with session: 9ac23147-2505-45c0-ab27-a02df26dd036","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.703689007Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
189082023-12-10T18:57:18.718Z{"msg":"cb8456ab-72ee-4ff3-9ae4-098f41b719e3 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.703746636Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
189092023-12-10T18:57:18.718Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.70377121Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
189102023-12-10T18:57:18.718Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.703798778Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
189112023-12-10T18:57:18.718Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.703863527Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
189122023-12-10T18:57:18.718Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.703887396Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
189132023-12-10T18:57:18.718Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.703912875Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
189142023-12-10T18:57:18.718Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.703965697Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
189152023-12-10T18:57:18.718Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.703989185Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
189162023-12-10T18:57:18.718Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.704126172Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
189172023-12-10T18:57:18.718Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.704189237Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
189182023-12-10T18:57:18.718ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
189192023-12-10T18:57:18.718Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.704448276Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
189202023-12-10T18:57:18.718Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:18.704476141Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1778,"session_id":"9ac23147-2505-45c0-ab27-a02df26dd036"}
189212023-12-10T18:57:18.719ZLoading write count information from file "/tmp/test_repair_verify.data"
189222023-12-10T18:57:18.719ZRun Repair workload
189232023-12-10T18:57:18.719Z01/30 Write block 83 len 28672 data: 17 17 15 18 16 17 15
189242023-12-10T18:57:18.719Z02/30 Write block 494 len 20480 data: 8 8 9 9 10
189252023-12-10T18:57:18.719Z03/30 Read block 450 len 28672
189262023-12-10T18:57:18.722Z04/30 Read block 235 len 36864
189272023-12-10T18:57:18.723Z05/30 Read block 495 len 8192
189282023-12-10T18:57:18.723Z06/30 Write block 495 len 4096 data: 9
189292023-12-10T18:57:18.723Z07/30 Write block 375 len 28672 data: 15 17 17 20 18 20 15
189302023-12-10T18:57:18.723Z08/30 Write block 520 len 36864 data: 9 10 10 11 12 11 13 12 12
189312023-12-10T18:57:18.723Z09/30 Read block 137 len 24576
189322023-12-10T18:57:18.724Z10/30 Read block 410 len 36864
189332023-12-10T18:57:18.727Z11/30 Write block 242 len 32768 data: 16 15 16 15 16 14 16 16
189342023-12-10T18:57:18.727Z12/30 Read block 266 len 4096
189352023-12-10T18:57:18.727Z13/30 Flush
189362023-12-10T18:57:18.730Z14/30 Write block 49 len 40960 data: 10 9 8 6 5 6 5 6 7 7
189372023-12-10T18:57:18.733Z15/30 Write block 329 len 16384 data: 12 13 11 14
189382023-12-10T18:57:18.733Z16/30 Read block 577 len 36864
189392023-12-10T18:57:18.736Z17/30 Write block 445 len 8192 data: 10 10
189402023-12-10T18:57:18.736Z18/30 Write block 0 len 12288 data: 3 4 5
189412023-12-10T18:57:18.739Z19/30 Write block 487 len 16384 data: 12 12 13 11
189422023-12-10T18:57:18.739Z20/30 Flush
189432023-12-10T18:57:18.742Z21/30 Write block 486 len 12288 data: 12 13 13
189442023-12-10T18:57:18.742Z22/30 Flush
189452023-12-10T18:57:18.745Z23/30 Flush
189462023-12-10T18:57:18.745Z24/30 Write block 276 len 16384 data: 14 15 14 11
189472023-12-10T18:57:18.748Z25/30 Write block 532 len 4096 data: 16
189482023-12-10T18:57:18.748Z26/30 Write block 113 len 32768 data: 14 12 14 15 14 14 13 14
189492023-12-10T18:57:18.748Z27/30 Read block 516 len 20480
189502023-12-10T18:57:18.751Z28/30 Write block 8 len 20480 data: 12 12 14 12 15
189512023-12-10T18:57:18.754Z29/30 Write block 23 len 40960 data: 11 12 11 11 8 7 6 9 12 15
189522023-12-10T18:57:18.754Z30/30 Read block 300 len 28672
189532023-12-10T18:57:18.758Z----------------------------------------------------------------
189542023-12-10T18:57:18.758Z Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30
189552023-12-10T18:57:18.758ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
189562023-12-10T18:57:18.758Z 1 Acked 1000 Write 7 Done Done Done false
189572023-12-10T18:57:18.758Z 2 Acked 1001 Write 5 Done Done Done false
189582023-12-10T18:57:18.758Z 3 Acked 1002 Read 7 Done Done Done false
189592023-12-10T18:57:18.759Z 4 Acked 1003 Read 9 Done Done Done false
189602023-12-10T18:57:18.759Z 5 Acked 1004 Read 2 Done Done Sent false
189612023-12-10T18:57:18.759Z 6 Acked 1005 Write 1 Done Done Sent false
189622023-12-10T18:57:18.759Z 7 Acked 1006 Write 7 Done Done Sent false
189632023-12-10T18:57:18.759Z 8 Acked 1007 Write 9 Done Done Sent false
189642023-12-10T18:57:18.759Z 9 Acked 1008 Read 6 Done Done Sent false
189652023-12-10T18:57:18.759Z 10 Acked 1009 Read 9 Done Done Sent false
189662023-12-10T18:57:18.759Z 11 Acked 1010 Write 8 Done Done Sent false
189672023-12-10T18:57:18.759Z 12 Acked 1011 Read 1 Done Done Sent false
189682023-12-10T18:57:18.759Z 13 Acked 1012 Flush 0 Done Done Sent false
189692023-12-10T18:57:18.759Z 14 Acked 1013 Write 10 Done Done Sent false
189702023-12-10T18:57:18.759Z 15 Acked 1014 Write 4 Done Done Sent false
189712023-12-10T18:57:18.759Z 16 Acked 1015 Read 9 Done Done Sent false
189722023-12-10T18:57:18.759Z 17 Acked 1016 Write 2 Done Done Sent false
189732023-12-10T18:57:18.759Z 18 Acked 1017 Write 3 Done Done Sent false
189742023-12-10T18:57:18.759Z 19 Acked 1018 Write 4 Done Done Sent false
189752023-12-10T18:57:18.759Z 20 Acked 1019 Flush 0 Done Done Sent false
189762023-12-10T18:57:18.762Z 21 Acked 1020 Write 3 Done Done Sent false
189772023-12-10T18:57:18.762Z 22 Acked 1021 Flush 0 Done Done Sent false
189782023-12-10T18:57:18.762Z 23 Acked 1022 Flush 0 Done Done Sent false
189792023-12-10T18:57:18.762Z 24 Acked 1023 Write 4 Done Done Sent false
189802023-12-10T18:57:18.762Z 25 Acked 1024 Write 1 Done Done Sent false
189812023-12-10T18:57:18.762Z 26 Acked 1025 Write 8 Done Done Sent false
189822023-12-10T18:57:18.762Z 27 Acked 1026 Read 5 Done Done Sent false
189832023-12-10T18:57:18.762Z 28 Acked 1027 Write 5 Done Done Sent false
189842023-12-10T18:57:18.762Z 29 Acked 1028 Write 10 Done Sent Sent false
189852023-12-10T18:57:18.763Z 30 Acked 1029 Read 7 Done Sent Sent false
189862023-12-10T18:57:18.763Z STATES DS:0 DS:1 DS:2 TOTAL
189872023-12-10T18:57:18.763Z New 0 0 0 0
189882023-12-10T18:57:18.763Z Sent 0 2 26 28
189892023-12-10T18:57:18.763Z Done 30 28 4 62
189902023-12-10T18:57:18.763Z Skipped 0 0 0 0
189912023-12-10T18:57:18.763Z Error 0 0 0 0
189922023-12-10T18:57:18.763ZLast Flush: 1022 1022 0
189932023-12-10T18:57:18.763ZDownstairs last five completed:
189942023-12-10T18:57:18.763ZUpstairs last five completed: 30 29 28 27 26 25
189952023-12-10T18:57:18.763ZWrote out file "/tmp/test_repair_verify.data"
189962023-12-10T18:57:18.766Z
189972023-12-10T18:57:18.766ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
189982023-12-10T18:57:18.793ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
189992023-12-10T18:57:18.793Z 0 000-019 154 154 147 268 268 256 T T F <---
190002023-12-10T18:57:18.796Z 1 020-039 151 151 151 262 262 262 T T F <---
190012023-12-10T18:57:18.796Z 2 040-059 154 154 153 268 268 266 F F F <---
190022023-12-10T18:57:18.796Z 3 060-079 151 151 151 262 262 262 F F F
190032023-12-10T18:57:18.797Z 4 080-099 154 154 148 267 267 259 F F T <---
190042023-12-10T18:57:18.797Z 5 100-119 152 152 152 263 263 263 T T F <---
190052023-12-10T18:57:18.797Z 6 120-139 136 136 136 235 235 235 T T F <---
190062023-12-10T18:57:18.797Z 7 140-159 152 152 152 263 263 263 F F F
190072023-12-10T18:57:18.797Z 8 160-179 152 152 152 265 265 265 F F F
190082023-12-10T18:57:18.797Z 9 180-199 150 150 150 261 261 261 F F F
190092023-12-10T18:57:18.797Z 10 200-219 152 152 152 265 265 265 F F F
190102023-12-10T18:57:18.797Z 11 220-239 152 152 152 263 263 263 F F F
190112023-12-10T18:57:18.797Z 12 240-259 154 154 153 267 267 266 F F F <---
190122023-12-10T18:57:18.797Z 13 260-279 140 140 140 243 243 243 T T F <---
190132023-12-10T18:57:18.797Z 14 280-299 153 153 153 266 266 266 F F F
190142023-12-10T18:57:18.797Z 15 300-319 148 148 148 259 259 259 F F F
190152023-12-10T18:57:18.797Z 16 320-339 154 154 149 268 268 260 F F F <---
190162023-12-10T18:57:18.797Z 17 340-359 148 148 148 257 257 257 F F F
190172023-12-10T18:57:18.797Z 18 360-379 154 154 148 267 267 259 F F F <---
190182023-12-10T18:57:18.797Z 19 380-399 154 154 152 267 267 263 F F F <---
190192023-12-10T18:57:18.797Z 20 400-419 150 150 150 261 261 261 F F F
190202023-12-10T18:57:18.798Z 21 420-439 153 153 153 266 266 266 F F F
190212023-12-10T18:57:18.798Z 22 440-459 154 154 152 268 268 265 F F F <---
190222023-12-10T18:57:18.798Z 23 460-479 152 152 152 263 263 263 F F F
190232023-12-10T18:57:18.798Z 24 480-499 154 154 142 269 269 245 F F T <---
190242023-12-10T18:57:18.799Z 25 500-519 140 140 140 240 240 240 F F F
190252023-12-10T18:57:18.799Z 26 520-539 154 154 151 267 267 262 T T F <---
190262023-12-10T18:57:18.799Z 27 540-559 148 148 148 259 259 259 F F F
190272023-12-10T18:57:18.799Z 28 560-579 153 153 153 266 266 266 F F F
190282023-12-10T18:57:18.799Z 29 580-599 146 146 146 253 253 253 F F F
190292023-12-10T18:57:18.799ZMax gen: 154, Max flush: 269
190302023-12-10T18:57:18.800ZError: Difference in extent metadata found!
190312023-12-10T18:57:18.803ZA Difference in extent metadata is expected here
190322023-12-10T18:57:18.803ZOn loop 77
190332023-12-10T18:57:18.803Z
190342023-12-10T18:57:18.803ZVerifying data now
190352023-12-10T18:57:25.134ZNo change after verify
190362023-12-10T18:57:25.134ZLoop: 77 Downstairs dump after verify (and repair):
190372023-12-10T18:57:25.156ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
190382023-12-10T18:57:25.156Z 0 000-019 155 155 155 270 270 270 F F F
190392023-12-10T18:57:25.159Z 1 020-039 155 155 155 270 270 270 F F F
190402023-12-10T18:57:25.160Z 2 040-059 154 154 154 268 268 268 F F F
190412023-12-10T18:57:25.160Z 3 060-079 151 151 151 262 262 262 F F F
190422023-12-10T18:57:25.160Z 4 080-099 154 154 154 267 267 267 F F F
190432023-12-10T18:57:25.160Z 5 100-119 155 155 155 270 270 270 F F F
190442023-12-10T18:57:25.160Z 6 120-139 155 155 155 270 270 270 F F F
190452023-12-10T18:57:25.160Z 7 140-159 152 152 152 263 263 263 F F F
190462023-12-10T18:57:25.160Z 8 160-179 152 152 152 265 265 265 F F F
190472023-12-10T18:57:25.160Z 9 180-199 150 150 150 261 261 261 F F F
190482023-12-10T18:57:25.160Z 10 200-219 152 152 152 265 265 265 F F F
190492023-12-10T18:57:25.160Z 11 220-239 152 152 152 263 263 263 F F F
190502023-12-10T18:57:25.160Z 12 240-259 154 154 154 267 267 267 F F F
190512023-12-10T18:57:25.160Z 13 260-279 155 155 155 270 270 270 F F F
190522023-12-10T18:57:25.160Z 14 280-299 153 153 153 266 266 266 F F F
190532023-12-10T18:57:25.160Z 15 300-319 148 148 148 259 259 259 F F F
190542023-12-10T18:57:25.160Z 16 320-339 154 154 154 268 268 268 F F F
190552023-12-10T18:57:25.160Z 17 340-359 148 148 148 257 257 257 F F F
190562023-12-10T18:57:25.161Z 18 360-379 154 154 154 267 267 267 F F F
190572023-12-10T18:57:25.161Z 19 380-399 154 154 154 267 267 267 F F F
190582023-12-10T18:57:25.161Z 20 400-419 150 150 150 261 261 261 F F F
190592023-12-10T18:57:25.161Z 21 420-439 153 153 153 266 266 266 F F F
190602023-12-10T18:57:25.161Z 22 440-459 154 154 154 268 268 268 F F F
190612023-12-10T18:57:25.161Z 23 460-479 152 152 152 263 263 263 F F F
190622023-12-10T18:57:25.161Z 24 480-499 154 154 154 269 269 269 F F F
190632023-12-10T18:57:25.161Z 25 500-519 140 140 140 240 240 240 F F F
190642023-12-10T18:57:25.161Z 26 520-539 155 155 155 270 270 270 F F F
190652023-12-10T18:57:25.161Z 27 540-559 148 148 148 259 259 259 F F F
190662023-12-10T18:57:25.161Z 28 560-579 153 153 153 266 266 266 F F F
190672023-12-10T18:57:25.161Z 29 580-599 146 146 146 253 253 253 F F F
190682023-12-10T18:57:25.161ZMax gen: 155, Max flush: 270
190692023-12-10T18:57:25.165Z
190702023-12-10T18:57:25.165ZBegin loop 78
190712023-12-10T18:57:25.165ZDownstairs to restart: 1
190722023-12-10T18:57:25.192Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.191794446Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786}
190732023-12-10T18:57:25.193Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.19224052Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786}
190742023-12-10T18:57:25.196Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.192305378Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786}
190752023-12-10T18:57:25.197Z{"msg":"Crucible 3b6f43b7-599f-44ff-8f7d-6a70d09f8e0e has session id: 65808d1c-5a61-415b-b170-5158ad44b76b","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.192394358Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
190762023-12-10T18:57:25.197Z{"msg":"Upstairs opts: Upstairs UUID: 3b6f43b7-599f-44ff-8f7d-6a70d09f8e0e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
190772023-12-10T18:57:25.197Z,"time":"2023-12-10T18:57:25.19249932Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","The guest has requested activation with gen:156
190782023-12-10T18:57:25.197Zpid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
190792023-12-10T18:57:25.197Z{"msg":"Crucible stats registered with UUID: 3b6f43b7-599f-44ff-8f7d-6a70d09f8e0e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.192701003Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
190802023-12-10T18:57:25.197Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.192753399Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b","task":"up_listen"}
190812023-12-10T18:57:25.197Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.19277979Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
190822023-12-10T18:57:25.197Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.192821442Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
190832023-12-10T18:57:25.197Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.192849522Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"looper":"2","session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
190842023-12-10T18:57:25.197Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.193048181Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"looper":"0","session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
190852023-12-10T18:57:25.197Z{"msg":"[2] 3b6f43b7-599f-44ff-8f7d-6a70d09f8e0e looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.193134268Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"looper":"2","session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
190862023-12-10T18:57:25.197Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.193178326Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"looper":"1","session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
190872023-12-10T18:57:25.197Z{"msg":"[0] 3b6f43b7-599f-44ff-8f7d-6a70d09f8e0e looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.193248838Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"looper":"0","session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
190882023-12-10T18:57:25.197Z{"msg":"[1] 3b6f43b7-599f-44ff-8f7d-6a70d09f8e0e looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.193293981Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"looper":"1","session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
190892023-12-10T18:57:25.197Z{"msg":"3b6f43b7-599f-44ff-8f7d-6a70d09f8e0e active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.193686953Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
190902023-12-10T18:57:25.197Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.193769642Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
190912023-12-10T18:57:25.199Z{"msg":"Set desired generation to :156","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.1938073Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
190922023-12-10T18:57:25.199Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.194081304Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
190932023-12-10T18:57:25.199Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.194159624Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
190942023-12-10T18:57:25.199Z{"msg":"[2] 3b6f43b7-599f-44ff-8f7d-6a70d09f8e0e (65808d1c-5a61-415b-b170-5158ad44b76b) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.194532002Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
190952023-12-10T18:57:25.199Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.194722041Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
190962023-12-10T18:57:25.199Z{"msg":"[2] client is_active_req TRUE, promote! session 65808d1c-5a61-415b-b170-5158ad44b76b","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.194894015Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
190972023-12-10T18:57:25.199Z{"msg":"[0] 3b6f43b7-599f-44ff-8f7d-6a70d09f8e0e (65808d1c-5a61-415b-b170-5158ad44b76b) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.19494302Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
190982023-12-10T18:57:25.200Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.194980547Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
190992023-12-10T18:57:25.200Z{"msg":"[0] client is_active_req TRUE, promote! session 65808d1c-5a61-415b-b170-5158ad44b76b","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.19502005Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191002023-12-10T18:57:25.206Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.195322093Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191012023-12-10T18:57:25.206Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.195410385Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191022023-12-10T18:57:25.206Z{"msg":"3b6f43b7-599f-44ff-8f7d-6a70d09f8e0e WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.195455709Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191032023-12-10T18:57:25.206Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.195526056Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191042023-12-10T18:57:25.206Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.195573958Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191052023-12-10T18:57:25.206Z{"msg":"3b6f43b7-599f-44ff-8f7d-6a70d09f8e0e WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.195665918Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191062023-12-10T18:57:25.206Z{"msg":"[1] 3b6f43b7-599f-44ff-8f7d-6a70d09f8e0e (65808d1c-5a61-415b-b170-5158ad44b76b) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.195708818Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191072023-12-10T18:57:25.206Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.195756842Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191082023-12-10T18:57:25.206Z{"msg":"[1] client is_active_req TRUE, promote! session 65808d1c-5a61-415b-b170-5158ad44b76b","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.195798258Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191092023-12-10T18:57:25.206Z{"msg":"[2] 3b6f43b7-599f-44ff-8f7d-6a70d09f8e0e (65808d1c-5a61-415b-b170-5158ad44b76b) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.196068658Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191102023-12-10T18:57:25.207Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.196129867Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191112023-12-10T18:57:25.207Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:25.196154673Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191122023-12-10T18:57:25.207Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.196196523Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191132023-12-10T18:57:25.207Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.196312712Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191142023-12-10T18:57:25.207Z{"msg":"3b6f43b7-599f-44ff-8f7d-6a70d09f8e0e WaitActive WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.196364086Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191152023-12-10T18:57:25.207Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.196387419Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191162023-12-10T18:57:25.207Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.196427642Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191172023-12-10T18:57:25.207Z{"msg":"[0] 3b6f43b7-599f-44ff-8f7d-6a70d09f8e0e (65808d1c-5a61-415b-b170-5158ad44b76b) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.196616072Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191182023-12-10T18:57:25.207Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.196645461Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191192023-12-10T18:57:25.207Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:25.196666061Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191202023-12-10T18:57:25.207Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.196708336Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191212023-12-10T18:57:25.207Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.196731395Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191222023-12-10T18:57:25.207Z{"msg":"3b6f43b7-599f-44ff-8f7d-6a70d09f8e0e WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.196772682Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191232023-12-10T18:57:25.207Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.196794935Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191242023-12-10T18:57:25.207Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.19683475Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191252023-12-10T18:57:25.207Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.196859685Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191262023-12-10T18:57:25.207Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.196889294Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191272023-12-10T18:57:25.207Z{"msg":"3b6f43b7-599f-44ff-8f7d-6a70d09f8e0e WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.196936512Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191282023-12-10T18:57:25.208Z{"msg":"[1] 3b6f43b7-599f-44ff-8f7d-6a70d09f8e0e (65808d1c-5a61-415b-b170-5158ad44b76b) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.197447697Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191292023-12-10T18:57:25.208Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.197503786Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191302023-12-10T18:57:25.208Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:25.197527128Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191312023-12-10T18:57:25.208Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.197547021Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191322023-12-10T18:57:25.208Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.197605008Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191332023-12-10T18:57:25.208Z{"msg":"3b6f43b7-599f-44ff-8f7d-6a70d09f8e0e WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.197631688Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191342023-12-10T18:57:25.208Z{"msg":"[0]R flush_numbers[0..12]: [270, 270, 268, 262, 267, 270, 270, 263, 265, 261, 265, 263]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.197658905Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191352023-12-10T18:57:25.208Z{"msg":"[0]R generation[0..12]: [155, 155, 154, 151, 154, 155, 155, 152, 152, 150, 152, 152]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.197714673Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191362023-12-10T18:57:25.208Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.197740984Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191372023-12-10T18:57:25.208Z{"msg":"[1]R flush_numbers[0..12]: [270, 270, 268, 262, 267, 270, 270, 263, 265, 261, 265, 263]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.19776974Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191382023-12-10T18:57:25.208ZThe guest has finished waiting for activation with:156
191392023-12-10T18:57:25.208Z{"msg":"Wait for a query_work_queue command to finish before sending IO
191402023-12-10T18:57:25.208Z[1]R generation[0..12]: [155, 155, 154, 151, 154, 155, 155, 152, 152, 150, 152, 152]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.197827319Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191412023-12-10T18:57:25.208Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.197853146Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191422023-12-10T18:57:25.208Z{"msg":"[2]R flush_numbers[0..12]: [270, 270, 268, 262, 267, 270, 270, 263, 265, 261, 265, 263]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.197879936Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191432023-12-10T18:57:25.208Z{"msg":"[2]R generation[0..12]: [155, 155, 154, 151, 154, 155, 155, 152, 152, 150, 152, 152]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.197940875Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191442023-12-10T18:57:25.209Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.197966862Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191452023-12-10T18:57:25.209Z{"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.197986577Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191462023-12-10T18:57:25.209Z{"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.198057692Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191472023-12-10T18:57:25.209Z{"msg":"Next flush: 271","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.198082068Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191482023-12-10T18:57:25.209Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.198102896Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191492023-12-10T18:57:25.209Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.198165299Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191502023-12-10T18:57:25.212Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.19818955Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191512023-12-10T18:57:25.212Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.198255892Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191522023-12-10T18:57:25.213Z{"msg":"3b6f43b7-599f-44ff-8f7d-6a70d09f8e0e is now active with session: 65808d1c-5a61-415b-b170-5158ad44b76b","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.198282626Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191532023-12-10T18:57:25.213Z{"msg":"3b6f43b7-599f-44ff-8f7d-6a70d09f8e0e Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.198306572Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191542023-12-10T18:57:25.213Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.198369158Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191552023-12-10T18:57:25.213Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.198392792Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191562023-12-10T18:57:25.213Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.198448094Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191572023-12-10T18:57:25.213Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.198472933Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191582023-12-10T18:57:25.213Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.198528479Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191592023-12-10T18:57:25.213Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.198553418Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191602023-12-10T18:57:25.213Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.198608543Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191612023-12-10T18:57:25.213Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.198698472Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191622023-12-10T18:57:25.213Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.198757468Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191632023-12-10T18:57:25.213Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.198810468Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191642023-12-10T18:57:25.213Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:25.198875916Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1786,"session_id":"65808d1c-5a61-415b-b170-5158ad44b76b"}
191652023-12-10T18:57:25.213ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
191662023-12-10T18:57:25.213ZLoading write count information from file "/tmp/test_repair_verify.data"
191672023-12-10T18:57:25.213ZRun Repair workload
191682023-12-10T18:57:25.213Z01/30 Write block 347 len 4096 data: 17
191692023-12-10T18:57:25.213Z02/30 Write block 376 len 12288 data: 18 18 21
191702023-12-10T18:57:25.217Z03/30 Write block 539 len 28672 data: 11 17 19 19 17 16 12
191712023-12-10T18:57:25.217Z04/30 Write block 54 len 40960 data: 7 6 7 8 8 6 8 9 11 12
191722023-12-10T18:57:25.217Z05/30 Read block 182 len 40960
191732023-12-10T18:57:25.217Z06/30 Write block 201 len 8192 data: 16 15
191742023-12-10T18:57:25.217Z07/30 Write block 68 len 4096 data: 16
191752023-12-10T18:57:25.217Z08/30 Read block 524 len 4096
191762023-12-10T18:57:25.217Z09/30 Write block 147 len 40960 data: 8 11 11 10 10 11 12 16 12 10
191772023-12-10T18:57:25.218Z10/30 Read block 550 len 24576
191782023-12-10T18:57:25.218Z11/30 Read block 295 len 16384
191792023-12-10T18:57:25.221Z12/30 Flush
191802023-12-10T18:57:25.223Z13/30 Write block 17 len 36864 data: 15 11 12 13 11 12 12 13 12
191812023-12-10T18:57:25.223Z14/30 Write block 289 len 8192 data: 14 11
191822023-12-10T18:57:25.237Z15/30 Flush
191832023-12-10T18:57:25.237Z16/30 Write block 240 len 32768 data: 16 16 17 16 17 16 17 15
191842023-12-10T18:57:25.237Z17/30 Write block 252 len 12288 data: 14 14 12
191852023-12-10T18:57:25.237Z18/30 Read block 416 len 4096
191862023-12-10T18:57:25.237Z19/30 Read block 529 len 36864
191872023-12-10T18:57:25.237Z20/30 Read block 439 len 12288
191882023-12-10T18:57:25.237Z21/30 Write block 0 len 16384 data: 4 5 6 4
191892023-12-10T18:57:25.237Z22/30 Flush
191902023-12-10T18:57:25.240Z23/30 Write block 408 len 28672 data: 14 11 11 9 8 7 8
191912023-12-10T18:57:25.240Z24/30 Write block 294 len 36864 data: 10 9 12 12 12 14 10 13 13
191922023-12-10T18:57:25.243Z25/30 Read block 328 len 12288
191932023-12-10T18:57:25.243Z26/30 Read block 273 len 20480
191942023-12-10T18:57:25.246Z27/30 Write block 349 len 16384 data: 18 15 15 17
191952023-12-10T18:57:25.246Z28/30 Write block 83 len 20480 data: 18 18 16 19 17
191962023-12-10T18:57:25.249Z29/30 Write block 502 len 24576 data: 13 14 10 11 12 11
191972023-12-10T18:57:25.249Z30/30 Read block 336 len 28672
191982023-12-10T18:57:25.252Z----------------------------------------------------------------
191992023-12-10T18:57:25.253Z Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:30
192002023-12-10T18:57:25.253ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
192012023-12-10T18:57:25.253Z 1 Acked 1000 Write 1 Done Done Done false
192022023-12-10T18:57:25.253Z 2 Acked 1001 Write 3 Done Done Done false
192032023-12-10T18:57:25.253Z 3 Acked 1002 Write 7 Done Done Done false
192042023-12-10T18:57:25.253Z 4 Acked 1003 Write 10 Done Done Done false
192052023-12-10T18:57:25.253Z 5 Acked 1004 Read 10 Done Done Done false
192062023-12-10T18:57:25.253Z 6 Acked 1005 Write 2 Done Done Done false
192072023-12-10T18:57:25.253Z 7 Acked 1006 Write 1 Done Done Done false
192082023-12-10T18:57:25.253Z 8 Acked 1007 Read 1 Done Done Done false
192092023-12-10T18:57:25.253Z 9 Acked 1008 Write 10 Done Done Done false
192102023-12-10T18:57:25.253Z 10 Acked 1009 Read 6 Done Sent Done false
192112023-12-10T18:57:25.253Z 11 Acked 1010 Read 4 Done Sent Done false
192122023-12-10T18:57:25.253Z 12 Acked 1011 Flush 0 Done Sent Done false
192132023-12-10T18:57:25.253Z 13 Acked 1012 Write 9 Done Sent Done false
192142023-12-10T18:57:25.253Z 14 Acked 1013 Write 2 Done Sent Done false
192152023-12-10T18:57:25.253Z 15 Acked 1014 Flush 0 Done Sent Done false
192162023-12-10T18:57:25.254Z 16 Acked 1015 Write 8 Done Sent Done false
192172023-12-10T18:57:25.254Z 17 Acked 1016 Write 3 Done Sent Done false
192182023-12-10T18:57:25.254Z 18 Acked 1017 Read 1 Done Sent Done false
192192023-12-10T18:57:25.254Z 19 Acked 1018 Read 9 Done Sent Done false
192202023-12-10T18:57:25.254Z 20 Acked 1019 Read 3 Done Sent Done false
192212023-12-10T18:57:25.254Z 21 Acked 1020 Write 4 Done Sent Done false
192222023-12-10T18:57:25.254Z 22 Acked 1021 Flush 0 Done Sent Done false
192232023-12-10T18:57:25.254Z 23 Acked 1022 Write 7 Done Sent Done false
192242023-12-10T18:57:25.254Z 24 Acked 1023 Write 9 Done Sent Done false
192252023-12-10T18:57:25.254Z 25 Acked 1024 Read 3 Done Sent Done false
192262023-12-10T18:57:25.254Z 26 Acked 1025 Read 5 Done Sent Done false
192272023-12-10T18:57:25.254Z 27 Acked 1026 Write 4 Done Sent Done false
192282023-12-10T18:57:25.254Z 28 Acked 1027 Write 5 Done Sent Done false
192292023-12-10T18:57:25.254Z 29 Acked 1028 Write 6 Done Sent Done false
192302023-12-10T18:57:25.255Z 30 Acked 1029 Read 7 Done Sent Done false
192312023-12-10T18:57:25.255Z STATES DS:0 DS:1 DS:2 TOTAL
192322023-12-10T18:57:25.255Z New 0 0 0 0
192332023-12-10T18:57:25.255Z Sent 0 21 0 21
192342023-12-10T18:57:25.255Z Done 30 9 30 69
192352023-12-10T18:57:25.255Z Skipped 0 0 0 0
192362023-12-10T18:57:25.255Z Error 0 0 0 0
192372023-12-10T18:57:25.255ZLast Flush: 1021 0 1021
192382023-12-10T18:57:25.255ZDownstairs last five completed:
192392023-12-10T18:57:25.255ZUpstairs last five completed: 30 29 28 27 26 25
192402023-12-10T18:57:25.256ZWrote out file "/tmp/test_repair_verify.data"
192412023-12-10T18:57:25.259Z
192422023-12-10T18:57:25.262ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
192432023-12-10T18:57:25.289ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
192442023-12-10T18:57:25.289Z 0 000-019 156 155 156 273 270 273 F F F <---
192452023-12-10T18:57:25.293Z 1 020-039 156 155 156 272 270 272 F F F <---
192462023-12-10T18:57:25.293Z 2 040-059 156 154 156 271 268 271 F T F <---
192472023-12-10T18:57:25.293Z 3 060-079 156 151 156 271 262 271 F T F <---
192482023-12-10T18:57:25.293Z 4 080-099 154 154 154 267 267 267 T F T <---
192492023-12-10T18:57:25.293Z 5 100-119 155 155 155 270 270 270 F F F
192502023-12-10T18:57:25.293Z 6 120-139 155 155 155 270 270 270 F F F
192512023-12-10T18:57:25.293Z 7 140-159 156 152 156 271 263 271 F T F <---
192522023-12-10T18:57:25.293Z 8 160-179 152 152 152 265 265 265 F F F
192532023-12-10T18:57:25.293Z 9 180-199 150 150 150 261 261 261 F F F
192542023-12-10T18:57:25.293Z 10 200-219 156 152 156 271 265 271 F T F <---
192552023-12-10T18:57:25.299Z 11 220-239 152 152 152 263 263 263 F F F
192562023-12-10T18:57:25.299Z 12 240-259 156 154 156 273 267 273 F F F <---
192572023-12-10T18:57:25.299Z 13 260-279 155 155 155 270 270 270 F F F
192582023-12-10T18:57:25.299Z 14 280-299 156 153 156 272 266 272 T F T <---
192592023-12-10T18:57:25.299Z 15 300-319 148 148 148 259 259 259 T F T <---
192602023-12-10T18:57:25.299Z 16 320-339 154 154 154 268 268 268 F F F
192612023-12-10T18:57:25.299Z 17 340-359 156 148 156 271 257 271 T T T <---
192622023-12-10T18:57:25.299Z 18 360-379 156 154 156 271 267 271 F T F <---
192632023-12-10T18:57:25.299Z 19 380-399 154 154 154 267 267 267 F F F
192642023-12-10T18:57:25.299Z 20 400-419 150 150 150 261 261 261 T F T <---
192652023-12-10T18:57:25.299Z 21 420-439 153 153 153 266 266 266 F F F
192662023-12-10T18:57:25.299Z 22 440-459 154 154 154 268 268 268 F F F
192672023-12-10T18:57:25.299Z 23 460-479 152 152 152 263 263 263 F F F
192682023-12-10T18:57:25.299Z 24 480-499 154 154 154 269 269 269 F F F
192692023-12-10T18:57:25.299Z 25 500-519 140 140 140 240 240 240 T F T <---
192702023-12-10T18:57:25.299Z 26 520-539 156 155 156 271 270 271 F T F <---
192712023-12-10T18:57:25.299Z 27 540-559 156 148 156 271 259 271 F T F <---
192722023-12-10T18:57:25.300Z 28 560-579 153 153 153 266 266 266 F F F
192732023-12-10T18:57:25.300Z 29 580-599 146 146 146 253 253 253 F F F
192742023-12-10T18:57:25.300ZMax gen: 156, Max flush: 273
192752023-12-10T18:57:25.300ZError: Difference in extent metadata found!
192762023-12-10T18:57:25.303ZA Difference in extent metadata is expected here
192772023-12-10T18:57:25.303ZOn loop 78
192782023-12-10T18:57:25.303Z
192792023-12-10T18:57:25.303ZVerifying data now
192802023-12-10T18:57:31.790ZNo change after verify
192812023-12-10T18:57:31.791ZLoop: 78 Downstairs dump after verify (and repair):
192822023-12-10T18:57:31.816ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
192832023-12-10T18:57:31.817Z 0 000-019 156 156 156 273 273 273 F F F
192842023-12-10T18:57:31.821Z 1 020-039 156 156 156 272 272 272 F F F
192852023-12-10T18:57:31.821Z 2 040-059 156 156 156 271 271 271 F F F
192862023-12-10T18:57:31.821Z 3 060-079 156 156 156 271 271 271 F F F
192872023-12-10T18:57:31.821Z 4 080-099 157 157 157 274 274 274 F F F
192882023-12-10T18:57:31.821Z 5 100-119 155 155 155 270 270 270 F F F
192892023-12-10T18:57:31.821Z 6 120-139 155 155 155 270 270 270 F F F
192902023-12-10T18:57:31.821Z 7 140-159 156 156 156 271 271 271 F F F
192912023-12-10T18:57:31.821Z 8 160-179 152 152 152 265 265 265 F F F
192922023-12-10T18:57:31.821Z 9 180-199 150 150 150 261 261 261 F F F
192932023-12-10T18:57:31.821Z 10 200-219 156 156 156 271 271 271 F F F
192942023-12-10T18:57:31.821Z 11 220-239 152 152 152 263 263 263 F F F
192952023-12-10T18:57:31.821Z 12 240-259 156 156 156 273 273 273 F F F
192962023-12-10T18:57:31.822Z 13 260-279 155 155 155 270 270 270 F F F
192972023-12-10T18:57:31.822Z 14 280-299 157 157 157 274 274 274 F F F
192982023-12-10T18:57:31.822Z 15 300-319 157 157 157 274 274 274 F F F
192992023-12-10T18:57:31.822Z 16 320-339 154 154 154 268 268 268 F F F
193002023-12-10T18:57:31.822Z 17 340-359 157 157 157 274 274 274 F F F
193012023-12-10T18:57:31.822Z 18 360-379 156 156 156 271 271 271 F F F
193022023-12-10T18:57:31.822Z 19 380-399 154 154 154 267 267 267 F F F
193032023-12-10T18:57:31.822Z 20 400-419 157 157 157 274 274 274 F F F
193042023-12-10T18:57:31.822Z 21 420-439 153 153 153 266 266 266 F F F
193052023-12-10T18:57:31.822Z 22 440-459 154 154 154 268 268 268 F F F
193062023-12-10T18:57:31.822Z 23 460-479 152 152 152 263 263 263 F F F
193072023-12-10T18:57:31.822Z 24 480-499 154 154 154 269 269 269 F F F
193082023-12-10T18:57:31.823Z 25 500-519 157 157 157 274 274 274 F F F
193092023-12-10T18:57:31.823Z 26 520-539 156 156 156 271 271 271 F F F
193102023-12-10T18:57:31.823Z 27 540-559 156 156 156 271 271 271 F F F
193112023-12-10T18:57:31.823Z 28 560-579 153 153 153 266 266 266 F F F
193122023-12-10T18:57:31.823Z 29 580-599 146 146 146 253 253 253 F F F
193132023-12-10T18:57:31.823ZMax gen: 157, Max flush: 274
193142023-12-10T18:57:31.823Z
193152023-12-10T18:57:31.823ZBegin loop 79
193162023-12-10T18:57:31.823ZDownstairs to restart: 1
193172023-12-10T18:57:31.848Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.845906843Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794}
193182023-12-10T18:57:31.848Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.84637537Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794}
193192023-12-10T18:57:31.848Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.846440766Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794}
193202023-12-10T18:57:31.853Z{"msg":"Crucible 14bb03f0-dc1e-4f7a-a7fa-3e0bc5128769 has session id: 652a51f4-7328-4106-958c-0576dac50f33","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.846480712Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193212023-12-10T18:57:31.853Z{"msg":"Upstairs opts: Upstairs UUID: 14bb03f0-dc1e-4f7a-a7fa-3e0bc5128769, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.846557659Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193222023-12-10T18:57:31.853Z{"msg":"Crucible stats registered with UUID: 14bb03f0-dc1e-4f7a-a7fa-3e0bc5128769","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.846617559Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193232023-12-10T18:57:31.853ZCrucible runtime is spawned
193242023-12-10T18:57:31.853Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.846816197Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33","task":"up_listen"}
193252023-12-10T18:57:31.853ZThe guest has requested activation with gen:158
193262023-12-10T18:57:31.853Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.846992471Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193272023-12-10T18:57:31.853Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.847135782Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193282023-12-10T18:57:31.853Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.847214236Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"looper":"1","session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193292023-12-10T18:57:31.853Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.847298522Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"looper":"0","session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193302023-12-10T18:57:31.853Z{"msg":"[1] 14bb03f0-dc1e-4f7a-a7fa-3e0bc5128769 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.847375762Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"looper":"1","session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193312023-12-10T18:57:31.854Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.847420143Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"looper":"2","session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193322023-12-10T18:57:31.854Z{"msg":"[0] 14bb03f0-dc1e-4f7a-a7fa-3e0bc5128769 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.84751372Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"looper":"0","session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193332023-12-10T18:57:31.854Z{"msg":"[2] 14bb03f0-dc1e-4f7a-a7fa-3e0bc5128769 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.847626777Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"looper":"2","session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193342023-12-10T18:57:31.854Z{"msg":"14bb03f0-dc1e-4f7a-a7fa-3e0bc5128769 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.847958741Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193352023-12-10T18:57:31.855Z{"msg":"Set desired generation to :158","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.848021217Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193362023-12-10T18:57:31.855Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.848112121Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193372023-12-10T18:57:31.855Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.848408155Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193382023-12-10T18:57:31.855Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.848488054Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193392023-12-10T18:57:31.855Z{"msg":"[0] 14bb03f0-dc1e-4f7a-a7fa-3e0bc5128769 (652a51f4-7328-4106-958c-0576dac50f33) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.848946357Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193402023-12-10T18:57:31.855Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.849006865Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193412023-12-10T18:57:31.855Z{"msg":"[2] 14bb03f0-dc1e-4f7a-a7fa-3e0bc5128769 (652a51f4-7328-4106-958c-0576dac50f33) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.849033968Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193422023-12-10T18:57:31.856Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.849054543Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193432023-12-10T18:57:31.856Z{"msg":"[0] client is_active_req TRUE, promote! session 652a51f4-7328-4106-958c-0576dac50f33","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.849123723Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193442023-12-10T18:57:31.856Z{"msg":"[2] client is_active_req TRUE, promote! session 652a51f4-7328-4106-958c-0576dac50f33","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.849170049Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193452023-12-10T18:57:31.856Z{"msg":"[1] 14bb03f0-dc1e-4f7a-a7fa-3e0bc5128769 (652a51f4-7328-4106-958c-0576dac50f33) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.849798744Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193462023-12-10T18:57:31.856Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.849877703Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193472023-12-10T18:57:31.856Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.849953726Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193482023-12-10T18:57:31.887Z{"msg":"[1] client is_active_req TRUE, promote! session 652a51f4-7328-4106-958c-0576dac50f33","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.850026132Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193492023-12-10T18:57:31.888Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.850113063Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193502023-12-10T18:57:31.888Z{"msg":"14bb03f0-dc1e-4f7a-a7fa-3e0bc5128769 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.85015786Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193512023-12-10T18:57:31.888Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.850195776Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193522023-12-10T18:57:31.888Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.850240846Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193532023-12-10T18:57:31.888Z{"msg":"14bb03f0-dc1e-4f7a-a7fa-3e0bc5128769 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.850275841Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193542023-12-10T18:57:31.903Z{"msg":"[0] 14bb03f0-dc1e-4f7a-a7fa-3e0bc5128769 (652a51f4-7328-4106-958c-0576dac50f33) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.850649586Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193552023-12-10T18:57:31.903Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.850703041Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193562023-12-10T18:57:31.903Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:31.850742204Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193572023-12-10T18:57:31.903Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.850788666Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193582023-12-10T18:57:31.904Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.850867576Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193592023-12-10T18:57:31.904Z{"msg":"14bb03f0-dc1e-4f7a-a7fa-3e0bc5128769 WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.85094586Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193602023-12-10T18:57:31.904Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.851080557Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193612023-12-10T18:57:31.904Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.851124885Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193622023-12-10T18:57:31.904Z{"msg":"[2] 14bb03f0-dc1e-4f7a-a7fa-3e0bc5128769 (652a51f4-7328-4106-958c-0576dac50f33) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.851170748Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193632023-12-10T18:57:31.904Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.851213225Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193642023-12-10T18:57:31.904Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:31.851249866Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193652023-12-10T18:57:31.904Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.851297565Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193662023-12-10T18:57:31.904Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.851321183Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193672023-12-10T18:57:31.904Z{"msg":"14bb03f0-dc1e-4f7a-a7fa-3e0bc5128769 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.851369269Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193682023-12-10T18:57:31.904Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.851391876Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193692023-12-10T18:57:31.904Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.851432367Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193702023-12-10T18:57:31.904Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.851472984Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193712023-12-10T18:57:31.904Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.851527997Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193722023-12-10T18:57:31.904Z{"msg":"14bb03f0-dc1e-4f7a-a7fa-3e0bc5128769 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.851552221Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193732023-12-10T18:57:31.904Z{"msg":"[1] 14bb03f0-dc1e-4f7a-a7fa-3e0bc5128769 (652a51f4-7328-4106-958c-0576dac50f33) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.852135936Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193742023-12-10T18:57:31.904Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.852215668Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193752023-12-10T18:57:31.904Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:31.852241745Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193762023-12-10T18:57:31.905Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.852284229Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193772023-12-10T18:57:31.905Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.852307677Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193782023-12-10T18:57:31.905Z{"msg":"14bb03f0-dc1e-4f7a-a7fa-3e0bc5128769 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.852349479Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193792023-12-10T18:57:31.905Z{"msg":"[0]R flush_numbers[0..12]: [273, 272, 271, 271, 274, 270, 270, 271, 265, 261, 271, 263]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.85237336Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193802023-12-10T18:57:31.905Z{"msg":"[0]R generation[0..12]: [156, 156, 156, 156, 157, 155, 155, 156, 152, 150, 156, 152]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.852415783Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193812023-12-10T18:57:31.905Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.852440476Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193822023-12-10T18:57:31.905Z{"msg":"[1]R flush_numbers[0..12]: [273, 272, 271, 271, 274, 270, 270, 271, 265, 261, 271, 263]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.852492433Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193832023-12-10T18:57:31.905Z{"msg":"[1]R generation[0..12]: [156, 156, 156, 156, 157, 155, 155, 156, 152, 150, 156, 152]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:158
193842023-12-10T18:57:31.906Z,"time":"2023-12-10T18:57:31.852516421Z","hostname"Wait for a query_work_queue command to finish before sending IO
193852023-12-10T18:57:31.906Z:"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193862023-12-10T18:57:31.906Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.852542546Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193872023-12-10T18:57:31.906Z{"msg":"[2]R flush_numbers[0..12]: [273, 272, 271, 271, 274, 270, 270, 271, 265, 261, 271, 263]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.85259429Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193882023-12-10T18:57:31.906Z{"msg":"[2]R generation[0..12]: [156, 156, 156, 156, 157, 155, 155, 156, 152, 150, 156, 152]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.852617972Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193892023-12-10T18:57:31.906Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.852672107Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193902023-12-10T18:57:31.906Z{"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.852713062Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193912023-12-10T18:57:31.906Z{"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.852785882Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193922023-12-10T18:57:31.906Z{"msg":"Next flush: 275","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.852832658Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193932023-12-10T18:57:31.906Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.852872946Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193942023-12-10T18:57:31.906Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.852943305Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193952023-12-10T18:57:31.907Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.852974543Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193962023-12-10T18:57:31.907Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.852997036Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193972023-12-10T18:57:31.907Z{"msg":"14bb03f0-dc1e-4f7a-a7fa-3e0bc5128769 is now active with session: 652a51f4-7328-4106-958c-0576dac50f33","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.853018912Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193982023-12-10T18:57:31.907Z{"msg":"14bb03f0-dc1e-4f7a-a7fa-3e0bc5128769 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.85307336Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
193992023-12-10T18:57:31.907Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.853096827Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
194002023-12-10T18:57:31.907Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.853125427Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
194012023-12-10T18:57:31.907Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.853182793Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
194022023-12-10T18:57:31.907Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.853207772Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
194032023-12-10T18:57:31.907Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.853232264Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
194042023-12-10T18:57:31.907Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
194052023-12-10T18:57:31.907Z30,"time":"2023-12-10T18:57:31.853354195Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
194062023-12-10T18:57:31.908Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.853408756Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
194072023-12-10T18:57:31.908Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.853432986Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
194082023-12-10T18:57:31.908Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.85345292Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
194092023-12-10T18:57:31.909Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.853473106Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
194102023-12-10T18:57:31.909Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:31.853513418Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1794,"session_id":"652a51f4-7328-4106-958c-0576dac50f33"}
194112023-12-10T18:57:31.909ZLoading write count information from file "/tmp/test_repair_verify.data"
194122023-12-10T18:57:31.909ZRun Repair workload
194132023-12-10T18:57:31.909Z01/30 Write block 158 len 8192 data: 9 11
194142023-12-10T18:57:31.909Z02/30 Write block 166 len 28672 data: 17 16 16 15 14 15 11
194152023-12-10T18:57:31.913Z03/30 Write block 81 len 40960 data: 17 18 19 19 17 20 18 18 16 12
194162023-12-10T18:57:31.913Z04/30 Read block 378 len 12288
194172023-12-10T18:57:31.913Z05/30 Flush
194182023-12-10T18:57:31.913Z06/30 Write block 83 len 4096 data: 20
194192023-12-10T18:57:31.913Z07/30 Read block 371 len 4096
194202023-12-10T18:57:31.913Z08/30 Write block 296 len 32768 data: 13 13 13 15 11 14 14 16
194212023-12-10T18:57:31.913Z09/30 Flush
194222023-12-10T18:57:31.913Z10/30 Flush
194232023-12-10T18:57:31.913Z11/30 Write block 274 len 16384 data: 11 14 15 16
194242023-12-10T18:57:31.913Z12/30 Read block 491 len 40960
194252023-12-10T18:57:31.915Z13/30 Read block 403 len 36864
194262023-12-10T18:57:31.915Z14/30 Read block 492 len 32768
194272023-12-10T18:57:31.915Z15/30 Read block 349 len 20480
194282023-12-10T18:57:31.915Z16/30 Read block 203 len 32768
194292023-12-10T18:57:31.916Z17/30 Write block 485 len 4096 data: 10
194302023-12-10T18:57:31.916Z18/30 Flush
194312023-12-10T18:57:31.916Z19/30 Write block 572 len 24576 data: 9 11 12 13 13 13
194322023-12-10T18:57:31.916Z20/30 Read block 16 len 16384
194332023-12-10T18:57:31.916Z21/30 Write block 223 len 40960 data: 14 14 12 16 15 14 14 20 19 18
194342023-12-10T18:57:31.916Z22/30 Flush
194352023-12-10T18:57:31.916Z23/30 Write block 151 len 36864 data: 11 12 13 17 13 11 11 10 12
194362023-12-10T18:57:31.916Z24/30 Read block 508 len 20480
194372023-12-10T18:57:31.916Z25/30 Read block 432 len 40960
194382023-12-10T18:57:31.916Z26/30 Read block 581 len 28672
194392023-12-10T18:57:31.916Z27/30 Write block 210 len 20480 data: 8 9 6 8 12
194402023-12-10T18:57:31.916Z28/30 Write block 335 len 32768 data: 10 11 11 9 12 11 12 13
194412023-12-10T18:57:31.916Z29/30 Read block 116 len 4096
194422023-12-10T18:57:31.916Z30/30 Read block 338 len 40960
194432023-12-10T18:57:31.920Z----------------------------------------------------------------
194442023-12-10T18:57:31.920Z Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30
194452023-12-10T18:57:31.920ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
194462023-12-10T18:57:31.920Z 1 Acked 1000 Write 2 Done Done Done false
194472023-12-10T18:57:31.920Z 2 Acked 1001 Write 7 Done Done Done false
194482023-12-10T18:57:31.920Z 3 Acked 1002 Write 10 Done Sent Done false
194492023-12-10T18:57:31.920Z 4 Acked 1003 Read 3 Done Sent Done false
194502023-12-10T18:57:31.921Z 5 Acked 1004 Flush 0 Done Sent Done false
194512023-12-10T18:57:31.921Z 6 Acked 1005 Write 1 Done Sent Done false
194522023-12-10T18:57:31.921Z 7 Acked 1006 Read 1 Done Sent Done false
194532023-12-10T18:57:31.921Z 8 Acked 1007 Write 8 Done Sent Done false
194542023-12-10T18:57:31.921Z 9 Acked 1008 Flush 0 Done Sent Done false
194552023-12-10T18:57:31.921Z 10 Acked 1009 Flush 0 Done Sent Done false
194562023-12-10T18:57:31.921Z 11 Acked 1010 Write 4 Done Sent Done false
194572023-12-10T18:57:31.921Z 12 Acked 1011 Read 10 Done Sent Done false
194582023-12-10T18:57:31.922Z 13 Acked 1012 Read 9 Done Sent Done false
194592023-12-10T18:57:31.922Z 14 Acked 1013 Read 8 Done Sent Done false
194602023-12-10T18:57:31.922Z 15 Acked 1014 Read 5 Done Sent Done false
194612023-12-10T18:57:31.922Z 16 Acked 1015 Read 8 Done Sent Done false
194622023-12-10T18:57:31.922Z 17 Acked 1016 Write 1 Done Sent Done false
194632023-12-10T18:57:31.922Z 18 Acked 1017 Flush 0 Done Sent Done false
194642023-12-10T18:57:31.922Z 19 Acked 1018 Write 6 Done Sent Done false
194652023-12-10T18:57:31.922Z 20 Acked 1019 Read 4 Done Sent Done false
194662023-12-10T18:57:31.922Z 21 Acked 1020 Write 10 Done Sent Done false
194672023-12-10T18:57:31.922Z 22 Acked 1021 Flush 0 Done Sent Done false
194682023-12-10T18:57:31.922Z 23 Acked 1022 Write 9 Done Sent Done false
194692023-12-10T18:57:31.922Z 24 Acked 1023 Read 5 Done Sent Done false
194702023-12-10T18:57:31.922Z 25 Acked 1024 Read 10 Done Sent Done false
194712023-12-10T18:57:31.922Z 26 Acked 1025 Read 7 Done Sent Done false
194722023-12-10T18:57:31.922Z 27 Acked 1026 Write 5 Done Sent Done false
194732023-12-10T18:57:31.922Z 28 Acked 1027 Write 8 Done Sent Done false
194742023-12-10T18:57:31.922Z 29 Acked 1028 Read 1 Done Sent Done false
194752023-12-10T18:57:31.922Z 30 Acked 1029 Read 10 Done Sent Done false
194762023-12-10T18:57:31.923Z STATES DS:0 DS:1 DS:2 TOTAL
194772023-12-10T18:57:31.923Z New 0 0 0 0
194782023-12-10T18:57:31.923Z Sent 0 28 0 28
194792023-12-10T18:57:31.923Z Done 30 2 30 62
194802023-12-10T18:57:31.923Z Skipped 0 0 0 0
194812023-12-10T18:57:31.923Z Error 0 0 0 0
194822023-12-10T18:57:31.923ZLast Flush: 1021 0 1021
194832023-12-10T18:57:31.923ZDownstairs last five completed:
194842023-12-10T18:57:31.923ZUpstairs last five completed: 30 29 28 27 26 25
194852023-12-10T18:57:31.923ZWrote out file "/tmp/test_repair_verify.data"
194862023-12-10T18:57:31.926Z
194872023-12-10T18:57:31.930ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
194882023-12-10T18:57:31.953ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
194892023-12-10T18:57:31.953Z 0 000-019 156 156 156 273 273 273 F F F
194902023-12-10T18:57:31.953Z 1 020-039 156 156 156 272 272 272 F F F
194912023-12-10T18:57:31.957Z 2 040-059 156 156 156 271 271 271 F F F
194922023-12-10T18:57:31.957Z 3 060-079 156 156 156 271 271 271 F F F
194932023-12-10T18:57:31.957Z 4 080-099 158 157 158 276 274 276 F F F <---
194942023-12-10T18:57:31.957Z 5 100-119 155 155 155 270 270 270 F F F
194952023-12-10T18:57:31.957Z 6 120-139 155 155 155 270 270 270 F F F
194962023-12-10T18:57:31.957Z 7 140-159 158 156 158 275 271 275 T T T <---
194972023-12-10T18:57:31.957Z 8 160-179 158 152 158 275 265 275 F T F <---
194982023-12-10T18:57:31.959Z 9 180-199 150 150 150 261 261 261 F F F
194992023-12-10T18:57:31.959Z 10 200-219 156 156 156 271 271 271 T F T <---
195002023-12-10T18:57:31.959Z 11 220-239 158 152 158 279 263 279 F F F <---
195012023-12-10T18:57:31.959Z 12 240-259 156 156 156 273 273 273 F F F
195022023-12-10T18:57:31.959Z 13 260-279 158 155 158 278 270 278 F F F <---
195032023-12-10T18:57:31.959Z 14 280-299 158 157 158 276 274 276 F F F <---
195042023-12-10T18:57:31.959Z 15 300-319 158 157 158 276 274 276 F F F <---
195052023-12-10T18:57:31.959Z 16 320-339 154 154 154 268 268 268 T F T <---
195062023-12-10T18:57:31.959Z 17 340-359 157 157 157 274 274 274 T F T <---
195072023-12-10T18:57:31.959Z 18 360-379 156 156 156 271 271 271 F F F
195082023-12-10T18:57:31.959Z 19 380-399 154 154 154 267 267 267 F F F
195092023-12-10T18:57:31.959Z 20 400-419 157 157 157 274 274 274 F F F
195102023-12-10T18:57:31.959Z 21 420-439 153 153 153 266 266 266 F F F
195112023-12-10T18:57:31.959Z 22 440-459 154 154 154 268 268 268 F F F
195122023-12-10T18:57:31.959Z 23 460-479 152 152 152 263 263 263 F F F
195132023-12-10T18:57:31.960Z 24 480-499 158 154 158 278 269 278 F F F <---
195142023-12-10T18:57:31.960Z 25 500-519 157 157 157 274 274 274 F F F
195152023-12-10T18:57:31.960Z 26 520-539 156 156 156 271 271 271 F F F
195162023-12-10T18:57:31.960Z 27 540-559 156 156 156 271 271 271 F F F
195172023-12-10T18:57:31.960Z 28 560-579 158 153 158 279 266 279 F F F <---
195182023-12-10T18:57:31.960Z 29 580-599 146 146 146 253 253 253 F F F
195192023-12-10T18:57:31.960ZMax gen: 158, Max flush: 279
195202023-12-10T18:57:31.960ZError: Difference in extent metadata found!
195212023-12-10T18:57:31.963ZA Difference in extent metadata is expected here
195222023-12-10T18:57:31.963ZOn loop 79
195232023-12-10T18:57:31.964Z
195242023-12-10T18:57:31.964ZVerifying data now
195252023-12-10T18:57:38.156ZNo change after verify
195262023-12-10T18:57:38.156ZLoop: 79 Downstairs dump after verify (and repair):
195272023-12-10T18:57:38.179ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
195282023-12-10T18:57:38.179Z 0 000-019 156 156 156 273 273 273 F F F
195292023-12-10T18:57:38.182Z 1 020-039 156 156 156 272 272 272 F F F
195302023-12-10T18:57:38.183Z 2 040-059 156 156 156 271 271 271 F F F
195312023-12-10T18:57:38.183Z 3 060-079 156 156 156 271 271 271 F F F
195322023-12-10T18:57:38.183Z 4 080-099 158 158 158 276 276 276 F F F
195332023-12-10T18:57:38.183Z 5 100-119 155 155 155 270 270 270 F F F
195342023-12-10T18:57:38.183Z 6 120-139 155 155 155 270 270 270 F F F
195352023-12-10T18:57:38.183Z 7 140-159 159 159 159 280 280 280 F F F
195362023-12-10T18:57:38.183Z 8 160-179 158 158 158 275 275 275 F F F
195372023-12-10T18:57:38.183Z 9 180-199 150 150 150 261 261 261 F F F
195382023-12-10T18:57:38.183Z 10 200-219 159 159 159 280 280 280 F F F
195392023-12-10T18:57:38.183Z 11 220-239 158 158 158 279 279 279 F F F
195402023-12-10T18:57:38.183Z 12 240-259 156 156 156 273 273 273 F F F
195412023-12-10T18:57:38.183Z 13 260-279 158 158 158 278 278 278 F F F
195422023-12-10T18:57:38.183Z 14 280-299 158 158 158 276 276 276 F F F
195432023-12-10T18:57:38.183Z 15 300-319 158 158 158 276 276 276 F F F
195442023-12-10T18:57:38.183Z 16 320-339 159 159 159 280 280 280 F F F
195452023-12-10T18:57:38.183Z 17 340-359 159 159 159 280 280 280 F F F
195462023-12-10T18:57:38.183Z 18 360-379 156 156 156 271 271 271 F F F
195472023-12-10T18:57:38.183Z 19 380-399 154 154 154 267 267 267 F F F
195482023-12-10T18:57:38.184Z 20 400-419 157 157 157 274 274 274 F F F
195492023-12-10T18:57:38.184Z 21 420-439 153 153 153 266 266 266 F F F
195502023-12-10T18:57:38.184Z 22 440-459 154 154 154 268 268 268 F F F
195512023-12-10T18:57:38.184Z 23 460-479 152 152 152 263 263 263 F F F
195522023-12-10T18:57:38.184Z 24 480-499 158 158 158 278 278 278 F F F
195532023-12-10T18:57:38.184Z 25 500-519 157 157 157 274 274 274 F F F
195542023-12-10T18:57:38.185Z 26 520-539 156 156 156 271 271 271 F F F
195552023-12-10T18:57:38.185Z 27 540-559 156 156 156 271 271 271 F F F
195562023-12-10T18:57:38.185Z 28 560-579 158 158 158 279 279 279 F F F
195572023-12-10T18:57:38.185Z 29 580-599 146 146 146 253 253 253 F F F
195582023-12-10T18:57:38.185ZMax gen: 159, Max flush: 280
195592023-12-10T18:57:38.188Z
195602023-12-10T18:57:38.188ZBegin loop 80
195612023-12-10T18:57:38.188ZDownstairs to restart: 1
195622023-12-10T18:57:38.215Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.212544879Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802}
195632023-12-10T18:57:38.215Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.213086704Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802}
195642023-12-10T18:57:38.219Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.213118293Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802}
195652023-12-10T18:57:38.219ZCrucible runtime is spawned
195662023-12-10T18:57:38.219Z{"msg":"Crucible 4e26f5c4-d06b-4365-8342-1c17eb29980c has session id: 6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.213195495Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195672023-12-10T18:57:38.219Z{"msg":"The guest has requested activation with gen:160
195682023-12-10T18:57:38.220ZUpstairs opts: Upstairs UUID: 4e26f5c4-d06b-4365-8342-1c17eb29980c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.213313883Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195692023-12-10T18:57:38.220Z{"msg":"Crucible stats registered with UUID: 4e26f5c4-d06b-4365-8342-1c17eb29980c","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.213403496Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195702023-12-10T18:57:38.220Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.21347423Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"looper":"0","session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195712023-12-10T18:57:38.220Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.213550987Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d","task":"up_listen"}
195722023-12-10T18:57:38.220Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.21362123Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195732023-12-10T18:57:38.220Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.213662323Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195742023-12-10T18:57:38.220Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.213729669Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"looper":"1","session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195752023-12-10T18:57:38.220Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.213776102Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"looper":"2","session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195762023-12-10T18:57:38.220Z{"msg":"[0] 4e26f5c4-d06b-4365-8342-1c17eb29980c looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.213834885Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"looper":"0","session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195772023-12-10T18:57:38.220Z{"msg":"4e26f5c4-d06b-4365-8342-1c17eb29980c active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.214035396Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195782023-12-10T18:57:38.220Z{"msg":"Set desired generation to :160","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.214144645Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195792023-12-10T18:57:38.220Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.214182344Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195802023-12-10T18:57:38.220Z{"msg":"[2] 4e26f5c4-d06b-4365-8342-1c17eb29980c looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.214206615Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"looper":"2","session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195812023-12-10T18:57:38.220Z{"msg":"[1] 4e26f5c4-d06b-4365-8342-1c17eb29980c looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.214277459Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"looper":"1","session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195822023-12-10T18:57:38.220Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.214414207Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195832023-12-10T18:57:38.220Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.214642298Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195842023-12-10T18:57:38.220Z{"msg":"[2] 4e26f5c4-d06b-4365-8342-1c17eb29980c (6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.21515698Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195852023-12-10T18:57:38.221Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.215220099Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195862023-12-10T18:57:38.221Z{"msg":"[2] client is_active_req TRUE, promote! session 6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.215246578Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195872023-12-10T18:57:38.221Z{"msg":"[0] 4e26f5c4-d06b-4365-8342-1c17eb29980c (6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.215448429Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195882023-12-10T18:57:38.221Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.215531157Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195892023-12-10T18:57:38.221Z{"msg":"[0] client is_active_req TRUE, promote! session 6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.215564681Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195902023-12-10T18:57:38.221Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.215964115Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195912023-12-10T18:57:38.221Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.216076794Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195922023-12-10T18:57:38.221Z{"msg":"4e26f5c4-d06b-4365-8342-1c17eb29980c WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.216102811Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195932023-12-10T18:57:38.221Z{"msg":"[1] 4e26f5c4-d06b-4365-8342-1c17eb29980c (6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.21622706Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195942023-12-10T18:57:38.221Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.216282523Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195952023-12-10T18:57:38.221Z{"msg":"[1] client is_active_req TRUE, promote! session 6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.216308895Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195962023-12-10T18:57:38.228Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.216347458Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195972023-12-10T18:57:38.228Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.216394461Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195982023-12-10T18:57:38.228Z{"msg":"4e26f5c4-d06b-4365-8342-1c17eb29980c WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.216436235Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
195992023-12-10T18:57:38.228Z{"msg":"[2] 4e26f5c4-d06b-4365-8342-1c17eb29980c (6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.216865399Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196002023-12-10T18:57:38.228Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.216929385Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196012023-12-10T18:57:38.228Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:38.216959033Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196022023-12-10T18:57:38.228Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.217019754Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196032023-12-10T18:57:38.229Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.217046475Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196042023-12-10T18:57:38.229Z{"msg":"4e26f5c4-d06b-4365-8342-1c17eb29980c WaitActive WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.217101047Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196052023-12-10T18:57:38.229Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.217123941Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196062023-12-10T18:57:38.229Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.21721122Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196072023-12-10T18:57:38.229Z{"msg":"[0] 4e26f5c4-d06b-4365-8342-1c17eb29980c (6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.217239748Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196082023-12-10T18:57:38.229Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.217276775Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196092023-12-10T18:57:38.229Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:38.217317519Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196102023-12-10T18:57:38.229Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.217409152Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196112023-12-10T18:57:38.229Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.217482662Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196122023-12-10T18:57:38.229Z{"msg":"4e26f5c4-d06b-4365-8342-1c17eb29980c WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.217530096Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196132023-12-10T18:57:38.229Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.217601036Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196142023-12-10T18:57:38.229Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.217642678Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196152023-12-10T18:57:38.230Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.217692712Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196162023-12-10T18:57:38.231Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.217721578Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196172023-12-10T18:57:38.232Z{"msg":"4e26f5c4-d06b-4365-8342-1c17eb29980c WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.217763737Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196182023-12-10T18:57:38.232Z{"msg":"[1] 4e26f5c4-d06b-4365-8342-1c17eb29980c (6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.218242217Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196192023-12-10T18:57:38.232Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.21832712Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196202023-12-10T18:57:38.232Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:38.218403068Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196212023-12-10T18:57:38.232Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.218444623Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196222023-12-10T18:57:38.232Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.218518396Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196232023-12-10T18:57:38.232Z{"msg":"4e26f5c4-d06b-4365-8342-1c17eb29980c WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.218558244Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196242023-12-10T18:57:38.232Z{"msg":"[0]R flush_numbers[0..12]: [273, 272, 271, 271, 276, 270, 270, 280, 275, 261, 280, 279]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.218597021Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196252023-12-10T18:57:38.232Z{"msg":"[0]R generation[0..12]: [156, 156, 156, 156, 158, 155, 155, 159, 158, 150, 159, 158]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.218637619Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid"The guest has finished waiting for activation with:160
196262023-12-10T18:57:38.232ZWait for a query_work_queue command to finish before sending IO
196272023-12-10T18:57:38.233Z:1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196282023-12-10T18:57:38.233Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.218782292Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196292023-12-10T18:57:38.233Z{"msg":"[1]R flush_numbers[0..12]: [273, 272, 271, 271, 276, 270, 270, 280, 275, 261, 280, 279]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.218864524Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196302023-12-10T18:57:38.233Z{"msg":"[1]R generation[0..12]: [156, 156, 156, 156, 158, 155, 155, 159, 158, 150, 159, 158]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.218911772Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196312023-12-10T18:57:38.233Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.218996274Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196322023-12-10T18:57:38.233Z{"msg":"[2]R flush_numbers[0..12]: [273, 272, 271, 271, 276, 270, 270, 280, 275, 261, 280, 279]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.219130013Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196332023-12-10T18:57:38.233Z{"msg":"[2]R generation[0..12]: [156, 156, 156, 156, 158, 155, 155, 159, 158, 150, 159, 158]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.219229021Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196342023-12-10T18:57:38.233Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.219273324Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196352023-12-10T18:57:38.234Z{"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.219329997Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196362023-12-10T18:57:38.234Z{"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.219354927Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196372023-12-10T18:57:38.234Z{"msg":"Next flush: 281","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.219414662Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196382023-12-10T18:57:38.234Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.219441599Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196392023-12-10T18:57:38.234Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.219521135Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196402023-12-10T18:57:38.234Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.219662558Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196412023-12-10T18:57:38.234Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.219714769Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196422023-12-10T18:57:38.234Z{"msg":"4e26f5c4-d06b-4365-8342-1c17eb29980c is now active with session: 6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.219758595Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196432023-12-10T18:57:38.234Z{"msg":"4e26f5c4-d06b-4365-8342-1c17eb29980c Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.219798885Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
196442023-12-10T18:57:38.234Zip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196452023-12-10T18:57:38.234Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.219984115Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196462023-12-10T18:57:38.234Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.22004324Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196472023-12-10T18:57:38.234Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.220067469Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196482023-12-10T18:57:38.234Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.220123962Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196492023-12-10T18:57:38.235Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.220152609Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196502023-12-10T18:57:38.236Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.220206384Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196512023-12-10T18:57:38.236Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.220233635Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196522023-12-10T18:57:38.236Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.220287846Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196532023-12-10T18:57:38.236Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.220316083Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196542023-12-10T18:57:38.236Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.220369386Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196552023-12-10T18:57:38.236Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:38.220393689Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1802,"session_id":"6a2ccc27-b6a5-4506-9ff8-cf7b92b1095d"}
196562023-12-10T18:57:38.236ZLoading write count information from file "/tmp/test_repair_verify.data"
196572023-12-10T18:57:38.236ZRun Repair workload
196582023-12-10T18:57:38.236Z01/30 Write block 233 len 40960 data: 15 18 18 15 14 13 17 17 17 18
196592023-12-10T18:57:38.236Z02/30 Read block 586 len 20480
196602023-12-10T18:57:38.240Z03/30 Read block 106 len 28672
196612023-12-10T18:57:38.240Z04/30 Read block 410 len 12288
196622023-12-10T18:57:38.240Z05/30 Write block 95 len 36864 data: 15 10 7 7 8 10 11 12 11
196632023-12-10T18:57:38.240Z06/30 Read block 309 len 12288
196642023-12-10T18:57:38.240Z07/30 Read block 459 len 32768
196652023-12-10T18:57:38.240Z08/30 Read block 433 len 24576
196662023-12-10T18:57:38.243Z09/30 Read block 323 len 20480
196672023-12-10T18:57:38.243Z10/30 Write block 429 len 4096 data: 10
196682023-12-10T18:57:38.246Z11/30 Write block 262 len 32768 data: 13 14 12 13 13 14 16 15
196692023-12-10T18:57:38.246Z12/30 Write block 142 len 40960 data: 6 5 6 6 9 9 12 12 11 12
196702023-12-10T18:57:38.246Z13/30 Write block 553 len 36864 data: 7 7 9 8 5 7 11 11 13
196712023-12-10T18:57:38.249Z14/30 Write block 394 len 20480 data: 9 10 9 9 10
196722023-12-10T18:57:38.249Z15/30 Flush
196732023-12-10T18:57:38.253Z16/30 Write block 155 len 12288 data: 14 12 12
196742023-12-10T18:57:38.256Z17/30 Write block 52 len 24576 data: 7 6 8 7 8 9
196752023-12-10T18:57:38.256Z18/30 Write block 420 len 32768 data: 11 13 14 14 16 13 13 14
196762023-12-10T18:57:38.256Z19/30 Read block 155 len 24576
196772023-12-10T18:57:38.259Z20/30 Write block 359 len 20480 data: 14 12 11 10 10
196782023-12-10T18:57:38.261Z21/30 Read block 200 len 16384
196792023-12-10T18:57:38.264Z22/30 Read block 121 len 28672
196802023-12-10T18:57:38.267Z23/30 Write block 445 len 24576 data: 11 11 10 12 12 12
196812023-12-10T18:57:38.267Z24/30 Read block 134 len 20480
196822023-12-10T18:57:38.270Z25/30 Read block 37 len 40960
196832023-12-10T18:57:38.276Z26/30 Read block 260 len 28672
196842023-12-10T18:57:38.279Z27/30 Write block 310 len 16384 data: 7 7 8 10
196852023-12-10T18:57:38.279Z28/30 Write block 132 len 12288 data: 7 7 6
196862023-12-10T18:57:38.279Z29/30 Read block 305 len 28672
196872023-12-10T18:57:38.283Z30/30 Read block 491 len 8192
196882023-12-10T18:57:38.283Z----------------------------------------------------------------
196892023-12-10T18:57:38.283Z Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30
196902023-12-10T18:57:38.283ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
196912023-12-10T18:57:38.283Z 1 Acked 1000 Write 10 Done Done Done false
196922023-12-10T18:57:38.283Z 2 Acked 1001 Read 5 Done Done Done false
196932023-12-10T18:57:38.283Z 3 Acked 1002 Read 7 Done Done Done false
196942023-12-10T18:57:38.283Z 4 Acked 1003 Read 3 Done Done Done false
196952023-12-10T18:57:38.283Z 5 Acked 1004 Write 9 Done Done Done false
196962023-12-10T18:57:38.283Z 6 Acked 1005 Read 3 Done Done Done false
196972023-12-10T18:57:38.283Z 7 Acked 1006 Read 8 Done Done Done false
196982023-12-10T18:57:38.283Z 8 Acked 1007 Read 6 Done Sent Done false
196992023-12-10T18:57:38.284Z 9 Acked 1008 Read 5 Done Sent Done false
197002023-12-10T18:57:38.284Z 10 Acked 1009 Write 1 Done Sent Done false
197012023-12-10T18:57:38.284Z 11 Acked 1010 Write 8 Done Sent Done false
197022023-12-10T18:57:38.284Z 12 Acked 1011 Write 10 Done Sent Done false
197032023-12-10T18:57:38.284Z 13 Acked 1012 Write 9 Done Sent Done false
197042023-12-10T18:57:38.284Z 14 Acked 1013 Write 5 Done Sent Done false
197052023-12-10T18:57:38.284Z 15 Acked 1014 Flush 0 Done Sent Done false
197062023-12-10T18:57:38.284Z 16 Acked 1015 Write 3 Done Sent Done false
197072023-12-10T18:57:38.284Z 17 Acked 1016 Write 6 Done Sent Done false
197082023-12-10T18:57:38.285Z 18 Acked 1017 Write 8 Done Sent Done false
197092023-12-10T18:57:38.285Z 19 Acked 1018 Read 6 Done Sent Done false
197102023-12-10T18:57:38.285Z 20 Acked 1019 Write 5 Done Sent Done false
197112023-12-10T18:57:38.285Z 21 Acked 1020 Read 4 Done Sent Done false
197122023-12-10T18:57:38.285Z 22 Acked 1021 Read 7 Done Sent Done false
197132023-12-10T18:57:38.285Z 23 Acked 1022 Write 6 Done Sent Done false
197142023-12-10T18:57:38.285Z 24 Acked 1023 Read 5 Done Sent Done false
197152023-12-10T18:57:38.285Z 25 Acked 1024 Read 10 Done Sent Done false
197162023-12-10T18:57:38.285Z 26 Acked 1025 Read 7 Done Sent Done false
197172023-12-10T18:57:38.285Z 27 Acked 1026 Write 4 Done Sent Done false
197182023-12-10T18:57:38.285Z 28 Acked 1027 Write 3 Done Sent Done false
197192023-12-10T18:57:38.285Z 29 Acked 1028 Read 7 Done Sent Done false
197202023-12-10T18:57:38.285Z 30 Acked 1029 Read 2 Sent Sent Done false
197212023-12-10T18:57:38.285Z STATES DS:0 DS:1 DS:2 TOTAL
197222023-12-10T18:57:38.285Z New 0 0 0 0
197232023-12-10T18:57:38.285Z Sent 1 23 0 24
197242023-12-10T18:57:38.286Z Done 29 7 30 66
197252023-12-10T18:57:38.286Z Skipped 0 0 0 0
197262023-12-10T18:57:38.286Z Error 0 0 0 0
197272023-12-10T18:57:38.286ZLast Flush: 1014 0 1014
197282023-12-10T18:57:38.286ZDownstairs last five completed:
197292023-12-10T18:57:38.286ZUpstairs last five completed: 30 29 28 27 26 25
197302023-12-10T18:57:38.286ZWrote out file "/tmp/test_repair_verify.data"
197312023-12-10T18:57:38.289Z
197322023-12-10T18:57:38.295ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
197332023-12-10T18:57:38.324ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
197342023-12-10T18:57:38.324Z 0 000-019 156 156 156 273 273 273 F F F
197352023-12-10T18:57:38.324Z 1 020-039 156 156 156 272 272 272 F F F
197362023-12-10T18:57:38.324Z 2 040-059 156 156 156 271 271 271 T F T <---
197372023-12-10T18:57:38.324Z 3 060-079 156 156 156 271 271 271 F F F
197382023-12-10T18:57:38.325Z 4 080-099 160 158 160 281 276 281 F T F <---
197392023-12-10T18:57:38.325Z 5 100-119 160 155 160 281 270 281 F T F <---
197402023-12-10T18:57:38.325Z 6 120-139 155 155 155 270 270 270 T F T <---
197412023-12-10T18:57:38.325Z 7 140-159 160 159 160 281 280 281 T F T <---
197422023-12-10T18:57:38.325Z 8 160-179 158 158 158 275 275 275 F F F
197432023-12-10T18:57:38.328Z 9 180-199 150 150 150 261 261 261 F F F
197442023-12-10T18:57:38.328Z 10 200-219 159 159 159 280 280 280 F F F
197452023-12-10T18:57:38.328Z 11 220-239 160 158 160 281 279 281 F T F <---
197462023-12-10T18:57:38.328Z 12 240-259 160 156 160 281 273 281 F T F <---
197472023-12-10T18:57:38.328Z 13 260-279 160 158 160 281 278 281 F F F <---
197482023-12-10T18:57:38.328Z 14 280-299 158 158 158 276 276 276 F F F
197492023-12-10T18:57:38.328Z 15 300-319 158 158 158 276 276 276 T F T <---
197502023-12-10T18:57:38.328Z 16 320-339 159 159 159 280 280 280 F F F
197512023-12-10T18:57:38.328Z 17 340-359 159 159 159 280 280 280 T F T <---
197522023-12-10T18:57:38.329Z 18 360-379 156 156 156 271 271 271 T F T <---
197532023-12-10T18:57:38.329Z 19 380-399 160 154 160 281 267 281 F F F <---
197542023-12-10T18:57:38.329Z 20 400-419 157 157 157 274 274 274 F F F
197552023-12-10T18:57:38.329Z 21 420-439 160 153 160 281 266 281 T F T <---
197562023-12-10T18:57:38.329Z 22 440-459 154 154 154 268 268 268 T F T <---
197572023-12-10T18:57:38.329Z 23 460-479 152 152 152 263 263 263 F F F
197582023-12-10T18:57:38.329Z 24 480-499 158 158 158 278 278 278 F F F
197592023-12-10T18:57:38.329Z 25 500-519 157 157 157 274 274 274 F F F
197602023-12-10T18:57:38.329Z 26 520-539 156 156 156 271 271 271 F F F
197612023-12-10T18:57:38.330Z 27 540-559 160 156 160 281 271 281 F F F <---
197622023-12-10T18:57:38.331Z 28 560-579 160 158 160 281 279 281 F F F <---
197632023-12-10T18:57:38.331Z 29 580-599 146 146 146 253 253 253 F F F
197642023-12-10T18:57:38.331ZMax gen: 160, Max flush: 281
197652023-12-10T18:57:38.331ZError: Difference in extent metadata found!
197662023-12-10T18:57:38.334ZA Difference in extent metadata is expected here
197672023-12-10T18:57:38.334ZOn loop 80
197682023-12-10T18:57:38.334Z
197692023-12-10T18:57:38.334ZVerifying data now
197702023-12-10T18:57:44.818ZNo change after verify
197712023-12-10T18:57:44.818ZLoop: 80 Downstairs dump after verify (and repair):
197722023-12-10T18:57:44.842ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
197732023-12-10T18:57:44.842Z 0 000-019 156 156 156 273 273 273 F F F
197742023-12-10T18:57:44.842Z 1 020-039 156 156 156 272 272 272 F F F
197752023-12-10T18:57:44.842Z 2 040-059 161 161 161 282 282 282 F F F
197762023-12-10T18:57:44.843Z 3 060-079 156 156 156 271 271 271 F F F
197772023-12-10T18:57:44.843Z 4 080-099 160 160 160 281 281 281 F F F
197782023-12-10T18:57:44.843Z 5 100-119 160 160 160 281 281 281 F F F
197792023-12-10T18:57:44.846Z 6 120-139 161 161 161 282 282 282 F F F
197802023-12-10T18:57:44.846Z 7 140-159 161 161 161 282 282 282 F F F
197812023-12-10T18:57:44.846Z 8 160-179 158 158 158 275 275 275 F F F
197822023-12-10T18:57:44.846Z 9 180-199 150 150 150 261 261 261 F F F
197832023-12-10T18:57:44.846Z 10 200-219 159 159 159 280 280 280 F F F
197842023-12-10T18:57:44.846Z 11 220-239 160 160 160 281 281 281 F F F
197852023-12-10T18:57:44.846Z 12 240-259 160 160 160 281 281 281 F F F
197862023-12-10T18:57:44.846Z 13 260-279 160 160 160 281 281 281 F F F
197872023-12-10T18:57:44.846Z 14 280-299 158 158 158 276 276 276 F F F
197882023-12-10T18:57:44.846Z 15 300-319 161 161 161 282 282 282 F F F
197892023-12-10T18:57:44.846Z 16 320-339 159 159 159 280 280 280 F F F
197902023-12-10T18:57:44.847Z 17 340-359 161 161 161 282 282 282 F F F
197912023-12-10T18:57:44.847Z 18 360-379 161 161 161 282 282 282 F F F
197922023-12-10T18:57:44.847Z 19 380-399 160 160 160 281 281 281 F F F
197932023-12-10T18:57:44.847Z 20 400-419 157 157 157 274 274 274 F F F
197942023-12-10T18:57:44.847Z 21 420-439 161 161 161 282 282 282 F F F
197952023-12-10T18:57:44.847Z 22 440-459 161 161 161 282 282 282 F F F
197962023-12-10T18:57:44.847Z 23 460-479 152 152 152 263 263 263 F F F
197972023-12-10T18:57:44.847Z 24 480-499 158 158 158 278 278 278 F F F
197982023-12-10T18:57:44.847Z 25 500-519 157 157 157 274 274 274 F F F
197992023-12-10T18:57:44.847Z 26 520-539 156 156 156 271 271 271 F F F
198002023-12-10T18:57:44.847Z 27 540-559 160 160 160 281 281 281 F F F
198012023-12-10T18:57:44.847Z 28 560-579 160 160 160 281 281 281 F F F
198022023-12-10T18:57:44.847Z 29 580-599 146 146 146 253 253 253 F F F
198032023-12-10T18:57:44.847ZMax gen: 161, Max flush: 282
198042023-12-10T18:57:44.852Z
198052023-12-10T18:57:44.852ZBegin loop 81
198062023-12-10T18:57:44.852ZDownstairs to restart: 2
198072023-12-10T18:57:44.880Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.875657645Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810}
198082023-12-10T18:57:44.888Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.876073855Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810}
198092023-12-10T18:57:44.888Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.876125581Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810}
198102023-12-10T18:57:44.888Z{"msg":"Crucible aaf568b1-85db-43ab-8d2f-b935cf4d95ec has session id: 0d5f5413-4b25-49a2-b549-a9d5bdbaf466","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.876149935Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198112023-12-10T18:57:44.889Z{"msg":"Upstairs opts: Upstairs UUID: aaf568b1-85db-43ab-8d2f-b935cf4d95ec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.876190712Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198122023-12-10T18:57:44.889Z{"msg":"Crucible stats registered with UUID: aaf568b1-85db-43ab-8d2f-b935cf4d95ec","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.876233633Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198132023-12-10T18:57:44.889ZCrucible runtime is spawned
198142023-12-10T18:57:44.889ZThe guest has requested activation with gen:162
198152023-12-10T18:57:44.889Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.876543134Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"looper":"0","session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198162023-12-10T18:57:44.889Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.876598646Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466","task":"up_listen"}
198172023-12-10T18:57:44.889Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.87667216Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198182023-12-10T18:57:44.889Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.876700476Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198192023-12-10T18:57:44.889Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.876874741Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"looper":"1","session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198202023-12-10T18:57:44.889Z{"msg":"[0] aaf568b1-85db-43ab-8d2f-b935cf4d95ec looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.876927344Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"looper":"0","session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198212023-12-10T18:57:44.889Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.877044569Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"looper":"2","session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198222023-12-10T18:57:44.889Z{"msg":"[1] aaf568b1-85db-43ab-8d2f-b935cf4d95ec looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.877126381Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"looper":"1","session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198232023-12-10T18:57:44.889Z{"msg":"[2] aaf568b1-85db-43ab-8d2f-b935cf4d95ec looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.877215825Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"looper":"2","session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198242023-12-10T18:57:44.889Z{"msg":"aaf568b1-85db-43ab-8d2f-b935cf4d95ec active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.877543667Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198252023-12-10T18:57:44.889Z{"msg":"Set desired generation to :162","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.877629579Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198262023-12-10T18:57:44.889Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.877736368Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198272023-12-10T18:57:44.889Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.878107243Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198282023-12-10T18:57:44.890Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.878161268Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198292023-12-10T18:57:44.890Z{"msg":"[0] aaf568b1-85db-43ab-8d2f-b935cf4d95ec (0d5f5413-4b25-49a2-b549-a9d5bdbaf466) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.878551329Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198302023-12-10T18:57:44.890Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.87860498Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198312023-12-10T18:57:44.890Z{"msg":"[0] client is_active_req TRUE, promote! session 0d5f5413-4b25-49a2-b549-a9d5bdbaf466","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.878628578Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198322023-12-10T18:57:44.890Z{"msg":"[1] aaf568b1-85db-43ab-8d2f-b935cf4d95ec (0d5f5413-4b25-49a2-b549-a9d5bdbaf466) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.878781973Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198332023-12-10T18:57:44.890Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.87883875Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198342023-12-10T18:57:44.890Z{"msg":"[1] client is_active_req TRUE, promote! session 0d5f5413-4b25-49a2-b549-a9d5bdbaf466","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.878878025Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198352023-12-10T18:57:44.890Z{"msg":"[2] aaf568b1-85db-43ab-8d2f-b935cf4d95ec (0d5f5413-4b25-49a2-b549-a9d5bdbaf466) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.879322503Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198362023-12-10T18:57:44.890Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.879432857Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198372023-12-10T18:57:44.890Z{"msg":"[2] client is_active_req TRUE, promote! session 0d5f5413-4b25-49a2-b549-a9d5bdbaf466","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.879483269Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198382023-12-10T18:57:44.890Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.879576685Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198392023-12-10T18:57:44.890Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.879629162Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198402023-12-10T18:57:44.891Z{"msg":"aaf568b1-85db-43ab-8d2f-b935cf4d95ec WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.879704505Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198412023-12-10T18:57:44.891Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.879762028Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198422023-12-10T18:57:44.891Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.879800147Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198432023-12-10T18:57:44.891Z{"msg":"aaf568b1-85db-43ab-8d2f-b935cf4d95ec WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.879842086Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198442023-12-10T18:57:44.891Z{"msg":"[1] aaf568b1-85db-43ab-8d2f-b935cf4d95ec (0d5f5413-4b25-49a2-b549-a9d5bdbaf466) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.880250517Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198452023-12-10T18:57:44.891Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.880331416Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198462023-12-10T18:57:44.891Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:44.880374388Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198472023-12-10T18:57:44.891Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.880433228Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198482023-12-10T18:57:44.891Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.880492866Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198492023-12-10T18:57:44.891Z{"msg":"aaf568b1-85db-43ab-8d2f-b935cf4d95ec WaitActive WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.880532826Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198502023-12-10T18:57:44.891Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.880569645Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198512023-12-10T18:57:44.892Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.880605527Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198522023-12-10T18:57:44.892Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.880671033Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198532023-12-10T18:57:44.892Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.880715846Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198542023-12-10T18:57:44.892Z{"msg":"aaf568b1-85db-43ab-8d2f-b935cf4d95ec WaitActive WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.880776616Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198552023-12-10T18:57:44.892Z{"msg":"[0] aaf568b1-85db-43ab-8d2f-b935cf4d95ec (0d5f5413-4b25-49a2-b549-a9d5bdbaf466) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.880820183Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198562023-12-10T18:57:44.892Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.880875194Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198572023-12-10T18:57:44.892Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:44.880913809Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198582023-12-10T18:57:44.892Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.880978554Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198592023-12-10T18:57:44.892Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.88101595Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198602023-12-10T18:57:44.892Z{"msg":"aaf568b1-85db-43ab-8d2f-b935cf4d95ec WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.881052896Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198612023-12-10T18:57:44.892Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.881116856Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198622023-12-10T18:57:44.892Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.881150815Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198632023-12-10T18:57:44.892Z{"msg":"[2] aaf568b1-85db-43ab-8d2f-b935cf4d95ec (0d5f5413-4b25-49a2-b549-a9d5bdbaf466) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.881191737Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198642023-12-10T18:57:44.892Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.881259146Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198652023-12-10T18:57:44.892Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:44.881288035Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198662023-12-10T18:57:44.892Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.881308333Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198672023-12-10T18:57:44.892Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.881358602Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198682023-12-10T18:57:44.892Z{"msg":"aaf568b1-85db-43ab-8d2f-b935cf4d95ec WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.881383593Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198692023-12-10T18:57:44.893Z{"msg":"[0]R flush_numbers[0..12]: [273, 272, 282, 271, 281, 281, 282, 282, 275, 261, 280, 281]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:162
198702023-12-10T18:57:44.893ZWait for a query_work_queue command to finish before sending IO
198712023-12-10T18:57:44.894Z,"time":"2023-12-10T18:57:44.881405901Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198722023-12-10T18:57:44.894Z{"msg":"[0]R generation[0..12]: [156, 156, 161, 156, 160, 160, 161, 161, 158, 150, 159, 160]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.881508089Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198732023-12-10T18:57:44.894Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.881543319Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198742023-12-10T18:57:44.894Z{"msg":"[1]R flush_numbers[0..12]: [273, 272, 282, 271, 281, 281, 282, 282, 275, 261, 280, 281]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.881632763Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198752023-12-10T18:57:44.894Z{"msg":"[1]R generation[0..12]: [156, 156, 161, 156, 160, 160, 161, 161, 158, 150, 159, 160]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.881675043Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198762023-12-10T18:57:44.894Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.881705548Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198772023-12-10T18:57:44.894Z{"msg":"[2]R flush_numbers[0..12]: [273, 272, 282, 271, 281, 281, 282, 282, 275, 261, 280, 281]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.881728181Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198782023-12-10T18:57:44.894Z{"msg":"[2]R generation[0..12]: [156, 156, 161, 156, 160, 160, 161, 161, 158, 150, 159, 160]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.881754632Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198792023-12-10T18:57:44.894Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.881820112Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198802023-12-10T18:57:44.894Z{"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.881845397Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198812023-12-10T18:57:44.895Z{"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.881886414Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198822023-12-10T18:57:44.895Z{"msg":"Next flush: 283","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.881908261Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198832023-12-10T18:57:44.895Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.88192793Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198842023-12-10T18:57:44.895Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.881988362Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198852023-12-10T18:57:44.899Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.882023949Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198862023-12-10T18:57:44.899Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.882067353Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198872023-12-10T18:57:44.899Z{"msg":"aaf568b1-85db-43ab-8d2f-b935cf4d95ec is now active with session: 0d5f5413-4b25-49a2-b549-a9d5bdbaf466","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.882091615Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198882023-12-10T18:57:44.899Z{"msg":"aaf568b1-85db-43ab-8d2f-b935cf4d95ec Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.882112335Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198892023-12-10T18:57:44.899Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.882132474Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198902023-12-10T18:57:44.899Z{"msg":"Set check for repair","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
198912023-12-10T18:57:44.899Zv":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.882226325Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198922023-12-10T18:57:44.900Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.882277592Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198932023-12-10T18:57:44.900Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.882301146Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198942023-12-10T18:57:44.901Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.882321433Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198952023-12-10T18:57:44.901Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.882362176Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198962023-12-10T18:57:44.902Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.882384425Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198972023-12-10T18:57:44.902Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.882403999Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198982023-12-10T18:57:44.902Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.882424021Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
198992023-12-10T18:57:44.902Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.882443553Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
199002023-12-10T18:57:44.902Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:44.882463148Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1810,"session_id":"0d5f5413-4b25-49a2-b549-a9d5bdbaf466"}
199012023-12-10T18:57:44.902ZLoading write count information from file "/tmp/test_repair_verify.data"
199022023-12-10T18:57:44.902ZRun Repair workload
199032023-12-10T18:57:44.902Z01/30 Write block 108 len 4096 data: 12
199042023-12-10T18:57:44.902Z02/30 Read block 497 len 40960
199052023-12-10T18:57:44.902Z03/30 Read block 315 len 20480
199062023-12-10T18:57:44.903Z04/30 Write block 447 len 40960 data: 11 13 13 13 14 13 13 11 11 11
199072023-12-10T18:57:44.903Z05/30 Read block 529 len 16384
199082023-12-10T18:57:44.903Z06/30 Write block 261 len 32768 data: 10 14 15 13 14 14 15 17
199092023-12-10T18:57:44.903Z07/30 Read block 177 len 12288
199102023-12-10T18:57:44.906Z08/30 Write block 405 len 24576 data: 14 13 13 15 12 12
199112023-12-10T18:57:44.906Z09/30 Write block 305 len 40960 data: 12 13 11 10 9 8 8 9 11 11
199122023-12-10T18:57:44.906Z10/30 Write block 587 len 8192 data: 11 12
199132023-12-10T18:57:44.906Z11/30 Write block 491 len 36864 data: 13 8 7 9 10 10 10 11 8
199142023-12-10T18:57:44.907Z12/30 Flush
199152023-12-10T18:57:44.910Z13/30 Write block 466 len 20480 data: 10 12 11 14 12
199162023-12-10T18:57:44.910Z14/30 Write block 496 len 32768 data: 11 11 12 9 12 14 14 15
199172023-12-10T18:57:44.913Z15/30 Read block 482 len 8192
199182023-12-10T18:57:44.913Z16/30 Read block 146 len 20480
199192023-12-10T18:57:44.916Z17/30 Read block 588 len 28672
199202023-12-10T18:57:44.919Z18/30 Flush
199212023-12-10T18:57:44.922Z19/30 Write block 543 len 20480 data: 18 17 13 11 11
199222023-12-10T18:57:44.922Z20/30 Read block 104 len 20480
199232023-12-10T18:57:44.924Z21/30 Read block 366 len 40960
199242023-12-10T18:57:44.928Z22/30 Read block 287 len 32768
199252023-12-10T18:57:44.930Z23/30 Write block 9 len 4096 data: 13
199262023-12-10T18:57:44.931Z24/30 Read block 158 len 32768
199272023-12-10T18:57:44.933Z25/30 Read block 170 len 28672
199282023-12-10T18:57:44.936Z26/30 Read block 79 len 40960
199292023-12-10T18:57:44.939Z27/30 Read block 427 len 16384
199302023-12-10T18:57:44.942Z28/30 Write block 85 len 28672 data: 18 21 19 19 17 13 10
199312023-12-10T18:57:44.942Z29/30 Write block 146 len 12288 data: 10 10 13
199322023-12-10T18:57:44.942Z30/30 Read block 345 len 20480
199332023-12-10T18:57:44.946Z----------------------------------------------------------------
199342023-12-10T18:57:44.946Z Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:18
199352023-12-10T18:57:44.946ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
199362023-12-10T18:57:44.946Z 13 Acked 1012 Write 5 Done Done Done false
199372023-12-10T18:57:44.946Z 14 Acked 1013 Write 8 Done Done Done false
199382023-12-10T18:57:44.946Z 15 Acked 1014 Read 2 Done Done Done false
199392023-12-10T18:57:44.946Z 16 Acked 1015 Read 5 Done Done Sent false
199402023-12-10T18:57:44.946Z 17 Acked 1016 Read 7 Done Done Sent false
199412023-12-10T18:57:44.946Z 18 Acked 1017 Flush 0 Done Done Sent false
199422023-12-10T18:57:44.946Z 19 Acked 1018 Write 5 Done Done Sent false
199432023-12-10T18:57:44.946Z 20 Acked 1019 Read 5 Done Done Sent false
199442023-12-10T18:57:44.946Z 21 Acked 1020 Read 10 Done Done Sent false
199452023-12-10T18:57:44.946Z 22 Acked 1021 Read 8 Done Done Sent false
199462023-12-10T18:57:44.947Z 23 Acked 1022 Write 1 Done Done Sent false
199472023-12-10T18:57:44.947Z 24 Acked 1023 Read 8 Done Done Sent false
199482023-12-10T18:57:44.947Z 25 Acked 1024 Read 7 Done Done Sent false
199492023-12-10T18:57:44.947Z 26 Acked 1025 Read 10 Done Done Sent false
199502023-12-10T18:57:44.947Z 27 Acked 1026 Read 4 Done Done Sent false
199512023-12-10T18:57:44.947Z 28 Acked 1027 Write 7 Done Done Sent false
199522023-12-10T18:57:44.947Z 29 Acked 1028 Write 3 Done Done Sent false
199532023-12-10T18:57:44.947Z 30 Acked 1029 Read 5 Done Done Sent false
199542023-12-10T18:57:44.947Z STATES DS:0 DS:1 DS:2 TOTAL
199552023-12-10T18:57:44.947Z New 0 0 0 0
199562023-12-10T18:57:44.947Z Sent 0 0 15 15
199572023-12-10T18:57:44.947Z Done 18 18 3 39
199582023-12-10T18:57:44.947Z Skipped 0 0 0 0
199592023-12-10T18:57:44.947Z Error 0 0 0 0
199602023-12-10T18:57:44.947ZLast Flush: 1017 1017 1011
199612023-12-10T18:57:44.947ZDownstairs last five completed: 1011 1010 1009 1008 1007 1006
199622023-12-10T18:57:44.947ZUpstairs last five completed: 30 29 28 27 26 25
199632023-12-10T18:57:44.948ZWrote out file "/tmp/test_repair_verify.data"
199642023-12-10T18:57:44.951Z
199652023-12-10T18:57:44.956ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
199662023-12-10T18:57:44.982ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
199672023-12-10T18:57:44.982Z 0 000-019 156 156 156 273 273 273 T T F <---
199682023-12-10T18:57:44.982Z 1 020-039 156 156 156 272 272 272 F F F
199692023-12-10T18:57:44.982Z 2 040-059 161 161 161 282 282 282 F F F
199702023-12-10T18:57:44.982Z 3 060-079 156 156 156 271 271 271 F F F
199712023-12-10T18:57:44.982Z 4 080-099 160 160 160 281 281 281 T T F <---
199722023-12-10T18:57:44.982Z 5 100-119 162 162 162 283 283 283 F F F
199732023-12-10T18:57:44.982Z 6 120-139 161 161 161 282 282 282 F F F
199742023-12-10T18:57:44.985Z 7 140-159 161 161 161 282 282 282 T T F <---
199752023-12-10T18:57:44.986Z 8 160-179 158 158 158 275 275 275 F F F
199762023-12-10T18:57:44.986Z 9 180-199 150 150 150 261 261 261 F F F
199772023-12-10T18:57:44.986Z 10 200-219 159 159 159 280 280 280 F F F
199782023-12-10T18:57:44.986Z 11 220-239 160 160 160 281 281 281 F F F
199792023-12-10T18:57:44.986Z 12 240-259 160 160 160 281 281 281 F F F
199802023-12-10T18:57:44.986Z 13 260-279 162 162 162 283 283 283 F F F
199812023-12-10T18:57:44.986Z 14 280-299 158 158 158 276 276 276 F F F
199822023-12-10T18:57:44.986Z 15 300-319 162 162 162 283 283 283 F F F
199832023-12-10T18:57:44.986Z 16 320-339 159 159 159 280 280 280 F F F
199842023-12-10T18:57:44.986Z 17 340-359 161 161 161 282 282 282 F F F
199852023-12-10T18:57:44.986Z 18 360-379 161 161 161 282 282 282 F F F
199862023-12-10T18:57:44.986Z 19 380-399 160 160 160 281 281 281 F F F
199872023-12-10T18:57:44.986Z 20 400-419 162 162 162 283 283 283 F F F
199882023-12-10T18:57:44.986Z 21 420-439 161 161 161 282 282 282 F F F
199892023-12-10T18:57:44.986Z 22 440-459 162 162 162 283 283 283 F F F
199902023-12-10T18:57:44.986Z 23 460-479 162 162 152 284 284 263 F F T <---
199912023-12-10T18:57:44.987Z 24 480-499 162 162 162 284 284 283 F F T <---
199922023-12-10T18:57:44.987Z 25 500-519 162 162 157 284 284 274 F F T <---
199932023-12-10T18:57:44.987Z 26 520-539 156 156 156 271 271 271 F F F
199942023-12-10T18:57:44.987Z 27 540-559 160 160 160 281 281 281 T T F <---
199952023-12-10T18:57:44.987Z 28 560-579 160 160 160 281 281 281 F F F
199962023-12-10T18:57:44.987Z 29 580-599 162 162 162 283 283 283 F F F
199972023-12-10T18:57:44.987ZMax gen: 162, Max flush: 284
199982023-12-10T18:57:44.987ZError: Difference in extent metadata found!
199992023-12-10T18:57:44.990ZA Difference in extent metadata is expected here
200002023-12-10T18:57:44.990ZOn loop 81
200012023-12-10T18:57:44.990Z
200022023-12-10T18:57:44.990ZVerifying data now
200032023-12-10T18:57:50.795ZNo change after verify
200042023-12-10T18:57:50.795ZLoop: 81 Downstairs dump after verify (and repair):
200052023-12-10T18:57:50.818ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
200062023-12-10T18:57:50.818Z 0 000-019 163 163 163 285 285 285 F F F
200072023-12-10T18:57:50.821Z 1 020-039 156 156 156 272 272 272 F F F
200082023-12-10T18:57:50.821Z 2 040-059 161 161 161 282 282 282 F F F
200092023-12-10T18:57:50.821Z 3 060-079 156 156 156 271 271 271 F F F
200102023-12-10T18:57:50.821Z 4 080-099 163 163 163 285 285 285 F F F
200112023-12-10T18:57:50.821Z 5 100-119 162 162 162 283 283 283 F F F
200122023-12-10T18:57:50.821Z 6 120-139 161 161 161 282 282 282 F F F
200132023-12-10T18:57:50.821Z 7 140-159 163 163 163 285 285 285 F F F
200142023-12-10T18:57:50.822Z 8 160-179 158 158 158 275 275 275 F F F
200152023-12-10T18:57:50.822Z 9 180-199 150 150 150 261 261 261 F F F
200162023-12-10T18:57:50.822Z 10 200-219 159 159 159 280 280 280 F F F
200172023-12-10T18:57:50.822Z 11 220-239 160 160 160 281 281 281 F F F
200182023-12-10T18:57:50.822Z 12 240-259 160 160 160 281 281 281 F F F
200192023-12-10T18:57:50.822Z 13 260-279 162 162 162 283 283 283 F F F
200202023-12-10T18:57:50.822Z 14 280-299 158 158 158 276 276 276 F F F
200212023-12-10T18:57:50.822Z 15 300-319 162 162 162 283 283 283 F F F
200222023-12-10T18:57:50.822Z 16 320-339 159 159 159 280 280 280 F F F
200232023-12-10T18:57:50.822Z 17 340-359 161 161 161 282 282 282 F F F
200242023-12-10T18:57:50.822Z 18 360-379 161 161 161 282 282 282 F F F
200252023-12-10T18:57:50.822Z 19 380-399 160 160 160 281 281 281 F F F
200262023-12-10T18:57:50.823Z 20 400-419 162 162 162 283 283 283 F F F
200272023-12-10T18:57:50.823Z 21 420-439 161 161 161 282 282 282 F F F
200282023-12-10T18:57:50.823Z 22 440-459 162 162 162 283 283 283 F F F
200292023-12-10T18:57:50.823Z 23 460-479 162 162 162 284 284 284 F F F
200302023-12-10T18:57:50.823Z 24 480-499 162 162 162 284 284 284 F F F
200312023-12-10T18:57:50.823Z 25 500-519 162 162 162 284 284 284 F F F
200322023-12-10T18:57:50.824Z 26 520-539 156 156 156 271 271 271 F F F
200332023-12-10T18:57:50.824Z 27 540-559 163 163 163 285 285 285 F F F
200342023-12-10T18:57:50.824Z 28 560-579 160 160 160 281 281 281 F F F
200352023-12-10T18:57:50.824Z 29 580-599 162 162 162 283 283 283 F F F
200362023-12-10T18:57:50.824ZMax gen: 163, Max flush: 285
200372023-12-10T18:57:50.827Z
200382023-12-10T18:57:50.827ZBegin loop 82
200392023-12-10T18:57:50.827ZDownstairs to restart: 0
200402023-12-10T18:57:50.853Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.85082806Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818}
200412023-12-10T18:57:50.857Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.851221209Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818}
200422023-12-10T18:57:50.857Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.851285739Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818}
200432023-12-10T18:57:50.857Z{"msg":"Crucible 030cb498-60bd-4b3d-abb0-50f82f0c1e49 has session id: 27d1a907-8f5b-40f2-ae19-843dc36b6230","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.851329953Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200442023-12-10T18:57:50.857Z{"msg":"Upstairs opts: Upstairs UUID: 030cb498-60bd-4b3d-abb0-50f82f0c1e49, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.851445756Z","hostname":"Crucible runtime is spawned
200452023-12-10T18:57:50.858Zip-10-150-1-104.us-west-2.compute.internal","pid":1818,"The guest has requested activation with gen:164
200462023-12-10T18:57:50.858Zsession_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200472023-12-10T18:57:50.858Z{"msg":"Crucible stats registered with UUID: 030cb498-60bd-4b3d-abb0-50f82f0c1e49","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.851663855Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200482023-12-10T18:57:50.858Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.851778939Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"looper":"1","session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200492023-12-10T18:57:50.858Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.851832288Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230","task":"up_listen"}
200502023-12-10T18:57:50.858Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.851873822Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200512023-12-10T18:57:50.858Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.851985207Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200522023-12-10T18:57:50.858Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.85204902Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"looper":"0","session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200532023-12-10T18:57:50.858Z{"msg":"[1] 030cb498-60bd-4b3d-abb0-50f82f0c1e49 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.852175458Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"looper":"1","session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200542023-12-10T18:57:50.859Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.852226017Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"looper":"2","session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200552023-12-10T18:57:50.859Z{"msg":"[0] 030cb498-60bd-4b3d-abb0-50f82f0c1e49 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.852309079Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"looper":"0","session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200562023-12-10T18:57:50.859Z{"msg":"[2] 030cb498-60bd-4b3d-abb0-50f82f0c1e49 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.852404566Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"looper":"2","session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200572023-12-10T18:57:50.859Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.852744526Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200582023-12-10T18:57:50.859Z{"msg":"030cb498-60bd-4b3d-abb0-50f82f0c1e49 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.852896858Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200592023-12-10T18:57:50.859Z{"msg":"Set desired generation to :164","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.852973609Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200602023-12-10T18:57:50.859Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.853239502Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200612023-12-10T18:57:50.859Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.8533387Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200622023-12-10T18:57:50.859Z{"msg":"[1] 030cb498-60bd-4b3d-abb0-50f82f0c1e49 (27d1a907-8f5b-40f2-ae19-843dc36b6230) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.85361655Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200632023-12-10T18:57:50.859Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.853680295Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200642023-12-10T18:57:50.859Z{"msg":"[1] client is_active_req TRUE, promote! session 27d1a907-8f5b-40f2-ae19-843dc36b6230","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.853788452Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200652023-12-10T18:57:50.859Z{"msg":"[2] 030cb498-60bd-4b3d-abb0-50f82f0c1e49 (27d1a907-8f5b-40f2-ae19-843dc36b6230) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.853915566Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200662023-12-10T18:57:50.859Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.853985373Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200672023-12-10T18:57:50.859Z{"msg":"[2] client is_active_req TRUE, promote! session 27d1a907-8f5b-40f2-ae19-843dc36b6230","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.854025421Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200682023-12-10T18:57:50.859Z{"msg":"[0] 030cb498-60bd-4b3d-abb0-50f82f0c1e49 (27d1a907-8f5b-40f2-ae19-843dc36b6230) New WaitActive WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.854065226Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200692023-12-10T18:57:50.859Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.854102112Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200702023-12-10T18:57:50.859Z{"msg":"[0] client is_active_req TRUE, promote! session 27d1a907-8f5b-40f2-ae19-843dc36b6230","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.8541377Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200712023-12-10T18:57:50.866Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.854325214Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200722023-12-10T18:57:50.866Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.854476597Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200732023-12-10T18:57:50.866Z{"msg":"030cb498-60bd-4b3d-abb0-50f82f0c1e49 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.854523424Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200742023-12-10T18:57:50.866Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.854595895Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200752023-12-10T18:57:50.867Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.854682026Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200762023-12-10T18:57:50.867Z{"msg":"030cb498-60bd-4b3d-abb0-50f82f0c1e49 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.854724421Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200772023-12-10T18:57:50.867Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.854935711Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200782023-12-10T18:57:50.867Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.855007156Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200792023-12-10T18:57:50.867Z{"msg":"030cb498-60bd-4b3d-abb0-50f82f0c1e49 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.855082673Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200802023-12-10T18:57:50.867Z{"msg":"[1] 030cb498-60bd-4b3d-abb0-50f82f0c1e49 (27d1a907-8f5b-40f2-ae19-843dc36b6230) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.855195064Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200812023-12-10T18:57:50.867Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.855277247Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200822023-12-10T18:57:50.867Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:50.855318901Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200832023-12-10T18:57:50.867Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.855381915Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200842023-12-10T18:57:50.870Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.855421173Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200852023-12-10T18:57:50.871Z{"msg":"030cb498-60bd-4b3d-abb0-50f82f0c1e49 WaitActive WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.855455904Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200862023-12-10T18:57:50.871Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.855491628Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200872023-12-10T18:57:50.871Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.85552801Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200882023-12-10T18:57:50.871Z{"msg":"[2] 030cb498-60bd-4b3d-abb0-50f82f0c1e49 (27d1a907-8f5b-40f2-ae19-843dc36b6230) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.85559105Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200892023-12-10T18:57:50.871Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.855631481Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200902023-12-10T18:57:50.871Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:50.855669297Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200912023-12-10T18:57:50.871Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.855730941Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200922023-12-10T18:57:50.871Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.855770361Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200932023-12-10T18:57:50.871Z{"msg":"030cb498-60bd-4b3d-abb0-50f82f0c1e49 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.855828699Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200942023-12-10T18:57:50.871Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.855868152Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200952023-12-10T18:57:50.872Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.855956027Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200962023-12-10T18:57:50.872Z{"msg":"[0] 030cb498-60bd-4b3d-abb0-50f82f0c1e49 (27d1a907-8f5b-40f2-ae19-843dc36b6230) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856000075Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200972023-12-10T18:57:50.872Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856059372Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200982023-12-10T18:57:50.872Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:50.856084722Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
200992023-12-10T18:57:50.872Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856125297Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201002023-12-10T18:57:50.872Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856147717Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201012023-12-10T18:57:50.872Z{"msg":"030cb498-60bd-4b3d-abb0-50f82f0c1e49 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856191661Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230The guest has finished waiting for activation with:164
201022023-12-10T18:57:50.872Z"}
201032023-12-10T18:57:50.873ZWait for a query_work_queue command to finish before sending IO
201042023-12-10T18:57:50.873Z{"msg":"[0]R flush_numbers[0..12]: [285, 272, 282, 271, 285, 283, 282, 285, 275, 261, 280, 281]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856227294Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201052023-12-10T18:57:50.873Z{"msg":"[0]R generation[0..12]: [163, 156, 161, 156, 163, 162, 161, 163, 158, 150, 159, 160]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856269959Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201062023-12-10T18:57:50.873Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856293867Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201072023-12-10T18:57:50.873Z{"msg":"[1]R flush_numbers[0..12]: [285, 272, 282, 271, 285, 283, 282, 285, 275, 261, 280, 281]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856315498Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201082023-12-10T18:57:50.873Z{"msg":"[1]R generation[0..12]: [163, 156, 161, 156, 163, 162, 161, 163, 158, 150, 159, 160]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856338373Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201092023-12-10T18:57:50.873Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856360558Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201102023-12-10T18:57:50.873Z{"msg":"[2]R flush_numbers[0..12]: [285, 272, 282, 271, 285, 283, 282, 285, 275, 261, 280, 281]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856382063Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201112023-12-10T18:57:50.873Z{"msg":"[2]R generation[0..12]: [163, 156, 161, 156, 163, 162, 161, 163, 158, 150, 159, 160]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856403699Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201122023-12-10T18:57:50.873Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856425243Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201132023-12-10T18:57:50.873Z{"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856449906Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201142023-12-10T18:57:50.873Z{"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.85647442Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201152023-12-10T18:57:50.873Z{"msg":"Next flush: 286","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856525335Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201162023-12-10T18:57:50.875Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856548522Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201172023-12-10T18:57:50.875Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856573451Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201182023-12-10T18:57:50.875Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856608245Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201192023-12-10T18:57:50.875Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856645774Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201202023-12-10T18:57:50.875Z{"msg":"030cb498-60bd-4b3d-abb0-50f82f0c1e49 is now active with session: 27d1a907-8f5b-40f2-ae19-843dc36b6230","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856687436Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201212023-12-10T18:57:50.875Z{"msg":"030cb498-60bd-4b3d-abb0-50f82f0c1e49 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.85671283Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201222023-12-10T18:57:50.875Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856756452Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201232023-12-10T18:57:50.875Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856778702Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201242023-12-10T18:57:50.875Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856818732Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201252023-12-10T18:57:50.875Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856844868Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201262023-12-10T18:57:50.875Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856910184Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201272023-12-10T18:57:50.877Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.856951952Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201282023-12-10T18:57:50.877Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.857008925Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201292023-12-10T18:57:50.877Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.857031879Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201302023-12-10T18:57:50.877Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.857084908Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201312023-12-10T18:57:50.877Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.857120303Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201322023-12-10T18:57:50.877Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:50.857169734Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1818,"session_id":"27d1a907-8f5b-40f2-ae19-843dc36b6230"}
201332023-12-10T18:57:50.877ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
201342023-12-10T18:57:50.877ZLoading write count information from file "/tmp/test_repair_verify.data"
201352023-12-10T18:57:50.877ZRun Repair workload
201362023-12-10T18:57:50.877Z01/30 Write block 516 len 32768 data: 9 9 9 10 10 11 11 12
201372023-12-10T18:57:50.877Z02/30 Read block 114 len 24576
201382023-12-10T18:57:50.881Z03/30 Read block 60 len 28672
201392023-12-10T18:57:50.881Z04/30 Write block 594 len 16384 data: 9 8 6 6
201402023-12-10T18:57:50.881Z05/30 Write block 520 len 40960 data: 11 12 12 13 13 12 14 13 13 15
201412023-12-10T18:57:50.881Z06/30 Flush
201422023-12-10T18:57:50.881Z07/30 Write block 570 len 12288 data: 13 12 10
201432023-12-10T18:57:50.881Z08/30 Read block 201 len 24576
201442023-12-10T18:57:50.881Z09/30 Write block 245 len 24576 data: 17 18 16 17 17 15
201452023-12-10T18:57:50.881Z10/30 Write block 18 len 36864 data: 12 13 14 12 13 13 14 13 12
201462023-12-10T18:57:50.882Z11/30 Read block 195 len 28672
201472023-12-10T18:57:50.885Z12/30 Read block 524 len 28672
201482023-12-10T18:57:50.885Z13/30 Read block 247 len 16384
201492023-12-10T18:57:50.888Z14/30 Read block 24 len 32768
201502023-12-10T18:57:50.891Z15/30 Write block 316 len 40960 data: 10 9 12 13 12 12 12 12 13 11
201512023-12-10T18:57:50.891Z16/30 Write block 393 len 12288 data: 10 10 11
201522023-12-10T18:57:50.894Z17/30 Write block 306 len 28672 data: 14 12 11 10 9 9 10
201532023-12-10T18:57:50.894Z18/30 Write block 130 len 16384 data: 11 10 8 8
201542023-12-10T18:57:50.898Z19/30 Write block 285 len 36864 data: 15 16 16 14 15 12 13 14 13
201552023-12-10T18:57:50.898Z20/30 Write block 71 len 20480 data: 15 12 15 16 14
201562023-12-10T18:57:50.898Z21/30 Write block 559 len 4096 data: 12
201572023-12-10T18:57:50.901Z22/30 Write block 561 len 40960 data: 14 16 13 13 13 14 14 11 12 14
201582023-12-10T18:57:50.901Z23/30 Read block 287 len 4096
201592023-12-10T18:57:50.901Z24/30 Read block 266 len 32768
201602023-12-10T18:57:50.907Z25/30 Flush
201612023-12-10T18:57:50.910Z26/30 Write block 12 len 20480 data: 16 17 18 17 14
201622023-12-10T18:57:50.910Z27/30 Read block 85 len 8192
201632023-12-10T18:57:50.910Z28/30 Read block 167 len 8192
201642023-12-10T18:57:50.913Z29/30 Read block 377 len 4096
201652023-12-10T18:57:50.913Z30/30 Read block 147 len 20480
201662023-12-10T18:57:50.917Z----------------------------------------------------------------
201672023-12-10T18:57:50.917Z Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:24
201682023-12-10T18:57:50.917ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
201692023-12-10T18:57:50.917Z 7 Acked 1006 Write 3 Done Done Done false
201702023-12-10T18:57:50.917Z 8 Acked 1007 Read 6 Sent Done Done false
201712023-12-10T18:57:50.917Z 9 Acked 1008 Write 6 Sent Done Done false
201722023-12-10T18:57:50.917Z 10 Acked 1009 Write 9 Sent Done Done false
201732023-12-10T18:57:50.918Z 11 Acked 1010 Read 7 Sent Done Done false
201742023-12-10T18:57:50.918Z 12 Acked 1011 Read 7 Sent Done Done false
201752023-12-10T18:57:50.918Z 13 Acked 1012 Read 4 Sent Done Done false
201762023-12-10T18:57:50.918Z 14 Acked 1013 Read 8 Sent Done Done false
201772023-12-10T18:57:50.918Z 15 Acked 1014 Write 10 Sent Done Done false
201782023-12-10T18:57:50.918Z 16 Acked 1015 Write 3 Sent Done Done false
201792023-12-10T18:57:50.918Z 17 Acked 1016 Write 7 Sent Done Done false
201802023-12-10T18:57:50.918Z 18 Acked 1017 Write 4 Sent Done Done false
201812023-12-10T18:57:50.918Z 19 Acked 1018 Write 9 Sent Done Done false
201822023-12-10T18:57:50.919Z 20 Acked 1019 Write 5 Sent Done Done false
201832023-12-10T18:57:50.919Z 21 Acked 1020 Write 1 Sent Done Done false
201842023-12-10T18:57:50.919Z 22 Acked 1021 Write 10 Sent Done Done false
201852023-12-10T18:57:50.919Z 23 Acked 1022 Read 1 Sent Done Done false
201862023-12-10T18:57:50.919Z 24 Acked 1023 Read 8 Sent Done Done false
201872023-12-10T18:57:50.919Z 25 Acked 1024 Flush 0 Sent Done Done false
201882023-12-10T18:57:50.919Z 26 Acked 1025 Write 5 Sent Done Done false
201892023-12-10T18:57:50.919Z 27 Acked 1026 Read 2 Sent Done Done false
201902023-12-10T18:57:50.919Z 28 Acked 1027 Read 2 Sent Done Done false
201912023-12-10T18:57:50.919Z 29 Acked 1028 Read 1 Sent Done Done false
201922023-12-10T18:57:50.919Z 30 Acked 1029 Read 5 Sent Sent Done false
201932023-12-10T18:57:50.919Z STATES DS:0 DS:1 DS:2 TOTAL
201942023-12-10T18:57:50.919Z New 0 0 0 0
201952023-12-10T18:57:50.919Z Sent 23 1 0 24
201962023-12-10T18:57:50.919Z Done 1 23 24 48
201972023-12-10T18:57:50.919Z Skipped 0 0 0 0
201982023-12-10T18:57:50.919Z Error 0 0 0 0
201992023-12-10T18:57:50.919ZLast Flush: 1005 1024 1024
202002023-12-10T18:57:50.920ZDownstairs last five completed: 1005 1004 1003 1002 1001 1000
202012023-12-10T18:57:50.920ZUpstairs last five completed: 30 29 28 27 26 25
202022023-12-10T18:57:50.920ZWrote out file "/tmp/test_repair_verify.data"
202032023-12-10T18:57:50.923Z
202042023-12-10T18:57:50.929ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
202052023-12-10T18:57:50.952ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
202062023-12-10T18:57:50.952Z 0 000-019 163 164 164 285 287 287 F T T <---
202072023-12-10T18:57:50.956Z 1 020-039 156 164 164 272 287 287 F F F <---
202082023-12-10T18:57:50.956Z 2 040-059 161 161 161 282 282 282 F F F
202092023-12-10T18:57:50.956Z 3 060-079 156 164 164 271 287 287 F F F <---
202102023-12-10T18:57:50.956Z 4 080-099 163 163 163 285 285 285 F F F
202112023-12-10T18:57:50.956Z 5 100-119 162 162 162 283 283 283 F F F
202122023-12-10T18:57:50.956Z 6 120-139 161 164 164 282 287 287 F F F <---
202132023-12-10T18:57:50.956Z 7 140-159 163 163 163 285 285 285 F F F
202142023-12-10T18:57:50.956Z 8 160-179 158 158 158 275 275 275 F F F
202152023-12-10T18:57:50.956Z 9 180-199 150 150 150 261 261 261 F F F
202162023-12-10T18:57:50.958Z 10 200-219 159 159 159 280 280 280 F F F
202172023-12-10T18:57:50.958Z 11 220-239 160 160 160 281 281 281 F F F
202182023-12-10T18:57:50.958Z 12 240-259 160 164 164 281 287 287 F F F <---
202192023-12-10T18:57:50.958Z 13 260-279 162 162 162 283 283 283 F F F
202202023-12-10T18:57:50.958Z 14 280-299 158 164 164 276 287 287 F F F <---
202212023-12-10T18:57:50.958Z 15 300-319 162 164 164 283 287 287 F F F <---
202222023-12-10T18:57:50.958Z 16 320-339 159 164 164 280 287 287 F F F <---
202232023-12-10T18:57:50.958Z 17 340-359 161 161 161 282 282 282 F F F
202242023-12-10T18:57:50.958Z 18 360-379 161 161 161 282 282 282 F F F
202252023-12-10T18:57:50.958Z 19 380-399 160 164 164 281 287 287 F F F <---
202262023-12-10T18:57:50.958Z 20 400-419 162 162 162 283 283 283 F F F
202272023-12-10T18:57:50.958Z 21 420-439 161 161 161 282 282 282 F F F
202282023-12-10T18:57:50.958Z 22 440-459 162 162 162 283 283 283 F F F
202292023-12-10T18:57:50.959Z 23 460-479 162 162 162 284 284 284 F F F
202302023-12-10T18:57:50.959Z 24 480-499 162 162 162 284 284 284 F F F
202312023-12-10T18:57:50.959Z 25 500-519 164 164 164 286 286 286 F F F
202322023-12-10T18:57:50.959Z 26 520-539 164 164 164 286 286 286 F F F
202332023-12-10T18:57:50.959Z 27 540-559 163 164 164 285 287 287 F F F <---
202342023-12-10T18:57:50.959Z 28 560-579 160 164 164 281 287 287 T F F <---
202352023-12-10T18:57:50.959Z 29 580-599 164 164 164 286 286 286 F F F
202362023-12-10T18:57:50.959ZMax gen: 164, Max flush: 287
202372023-12-10T18:57:50.959ZError: Difference in extent metadata found!
202382023-12-10T18:57:50.962ZA Difference in extent metadata is expected here
202392023-12-10T18:57:50.962ZOn loop 82
202402023-12-10T18:57:50.962Z
202412023-12-10T18:57:50.962ZVerifying data now
202422023-12-10T18:57:56.068ZNo change after verify
202432023-12-10T18:57:56.068ZLoop: 82 Downstairs dump after verify (and repair):
202442023-12-10T18:57:56.091ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
202452023-12-10T18:57:56.091Z 0 000-019 165 165 165 288 288 288 F F F
202462023-12-10T18:57:56.094Z 1 020-039 164 164 164 287 287 287 F F F
202472023-12-10T18:57:56.094Z 2 040-059 161 161 161 282 282 282 F F F
202482023-12-10T18:57:56.094Z 3 060-079 164 164 164 287 287 287 F F F
202492023-12-10T18:57:56.095Z 4 080-099 163 163 163 285 285 285 F F F
202502023-12-10T18:57:56.095Z 5 100-119 162 162 162 283 283 283 F F F
202512023-12-10T18:57:56.095Z 6 120-139 164 164 164 287 287 287 F F F
202522023-12-10T18:57:56.095Z 7 140-159 163 163 163 285 285 285 F F F
202532023-12-10T18:57:56.095Z 8 160-179 158 158 158 275 275 275 F F F
202542023-12-10T18:57:56.095Z 9 180-199 150 150 150 261 261 261 F F F
202552023-12-10T18:57:56.095Z 10 200-219 159 159 159 280 280 280 F F F
202562023-12-10T18:57:56.095Z 11 220-239 160 160 160 281 281 281 F F F
202572023-12-10T18:57:56.095Z 12 240-259 164 164 164 287 287 287 F F F
202582023-12-10T18:57:56.095Z 13 260-279 162 162 162 283 283 283 F F F
202592023-12-10T18:57:56.095Z 14 280-299 164 164 164 287 287 287 F F F
202602023-12-10T18:57:56.096Z 15 300-319 164 164 164 287 287 287 F F F
202612023-12-10T18:57:56.096Z 16 320-339 164 164 164 287 287 287 F F F
202622023-12-10T18:57:56.096Z 17 340-359 161 161 161 282 282 282 F F F
202632023-12-10T18:57:56.096Z 18 360-379 161 161 161 282 282 282 F F F
202642023-12-10T18:57:56.096Z 19 380-399 164 164 164 287 287 287 F F F
202652023-12-10T18:57:56.096Z 20 400-419 162 162 162 283 283 283 F F F
202662023-12-10T18:57:56.096Z 21 420-439 161 161 161 282 282 282 F F F
202672023-12-10T18:57:56.096Z 22 440-459 162 162 162 283 283 283 F F F
202682023-12-10T18:57:56.096Z 23 460-479 162 162 162 284 284 284 F F F
202692023-12-10T18:57:56.096Z 24 480-499 162 162 162 284 284 284 F F F
202702023-12-10T18:57:56.096Z 25 500-519 164 164 164 286 286 286 F F F
202712023-12-10T18:57:56.096Z 26 520-539 164 164 164 286 286 286 F F F
202722023-12-10T18:57:56.096Z 27 540-559 164 164 164 287 287 287 F F F
202732023-12-10T18:57:56.096Z 28 560-579 164 164 164 287 287 287 F F F
202742023-12-10T18:57:56.096Z 29 580-599 164 164 164 286 286 286 F F F
202752023-12-10T18:57:56.097ZMax gen: 165, Max flush: 288
202762023-12-10T18:57:56.100Z
202772023-12-10T18:57:56.100ZBegin loop 83
202782023-12-10T18:57:56.100ZDownstairs to restart: 1
202792023-12-10T18:57:56.126Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.124004916Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826}
202802023-12-10T18:57:56.126Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.124484965Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826}
202812023-12-10T18:57:56.131Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.124535927Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826}
202822023-12-10T18:57:56.132Z{"msg":"Crucible d2a74b43-bb99-4a51-8053-d9fa2b57c4cd has session id: b7f53597-227a-4284-97e3-4630087c3d27","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.124564212Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
202832023-12-10T18:57:56.132Z{"msg":"Upstairs opts: Upstairs UUID: d2a74b43-bb99-4a51-8053-d9fa2b57c4cd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.124619668Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
202842023-12-10T18:57:56.132Z{"msg":"Crucible stats registered with UUID: d2a74b43-bb99-4a51-8053-d9fa2b57c4cd","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.124666529Z"Crucible runtime is spawned
202852023-12-10T18:57:56.132Z,"hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
202862023-12-10T18:57:56.132Z{"msg":"up_listen starts",The guest has requested activation with gen:166
202872023-12-10T18:57:56.132Z"v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.124849669Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27","task":"up_listen"}
202882023-12-10T18:57:56.132Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.124931509Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
202892023-12-10T18:57:56.132Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.124973063Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"looper":"1","session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
202902023-12-10T18:57:56.132Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.125021399Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
202912023-12-10T18:57:56.132Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.125058167Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"looper":"0","session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
202922023-12-10T18:57:56.132Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.12509788Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"looper":"2","session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
202932023-12-10T18:57:56.132Z{"msg":"[2] d2a74b43-bb99-4a51-8053-d9fa2b57c4cd looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.125135665Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"looper":"2","session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
202942023-12-10T18:57:56.132Z{"msg":"d2a74b43-bb99-4a51-8053-d9fa2b57c4cd active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.12517321Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
202952023-12-10T18:57:56.132Z{"msg":"Set desired generation to :166","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.125210257Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
202962023-12-10T18:57:56.132Z{"msg":"[1] d2a74b43-bb99-4a51-8053-d9fa2b57c4cd looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.12527267Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"looper":"1","session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
202972023-12-10T18:57:56.132Z{"msg":"[0] d2a74b43-bb99-4a51-8053-d9fa2b57c4cd looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.125371659Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"looper":"0","session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
202982023-12-10T18:57:56.133Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.125577199Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
202992023-12-10T18:57:56.133Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.125979264Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203002023-12-10T18:57:56.133Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.126100725Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203012023-12-10T18:57:56.134Z{"msg":"[0] d2a74b43-bb99-4a51-8053-d9fa2b57c4cd (b7f53597-227a-4284-97e3-4630087c3d27) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.126574036Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203022023-12-10T18:57:56.134Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.126652862Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203032023-12-10T18:57:56.134Z{"msg":"[0] client is_active_req TRUE, promote! session b7f53597-227a-4284-97e3-4630087c3d27","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.126679532Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203042023-12-10T18:57:56.134Z{"msg":"[2] d2a74b43-bb99-4a51-8053-d9fa2b57c4cd (b7f53597-227a-4284-97e3-4630087c3d27) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.126865218Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203052023-12-10T18:57:56.134Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.126937701Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203062023-12-10T18:57:56.134Z{"msg":"[2] client is_active_req TRUE, promote! session b7f53597-227a-4284-97e3-4630087c3d27","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.126969416Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203072023-12-10T18:57:56.134Z{"msg":"[1] d2a74b43-bb99-4a51-8053-d9fa2b57c4cd (b7f53597-227a-4284-97e3-4630087c3d27) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.12750409Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203082023-12-10T18:57:56.135Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.127558409Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203092023-12-10T18:57:56.135Z{"msg":"[1] client is_active_req TRUE, promote! session b7f53597-227a-4284-97e3-4630087c3d27","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.127583092Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203102023-12-10T18:57:56.135Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.127699697Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203112023-12-10T18:57:56.135Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.127729256Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203122023-12-10T18:57:56.135Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.127753007Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203132023-12-10T18:57:56.135Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.127798816Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203142023-12-10T18:57:56.135Z{"msg":"d2a74b43-bb99-4a51-8053-d9fa2b57c4cd WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.12782269Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203152023-12-10T18:57:56.135Z{"msg":"d2a74b43-bb99-4a51-8053-d9fa2b57c4cd WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.127843706Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203162023-12-10T18:57:56.141Z{"msg":"[2] d2a74b43-bb99-4a51-8053-d9fa2b57c4cd (b7f53597-227a-4284-97e3-4630087c3d27) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.128433959Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203172023-12-10T18:57:56.141Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.128517951Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203182023-12-10T18:57:56.141Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:56.128543876Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203192023-12-10T18:57:56.141Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.128566268Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203202023-12-10T18:57:56.141Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.128621735Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203212023-12-10T18:57:56.141Z{"msg":"[0] d2a74b43-bb99-4a51-8053-d9fa2b57c4cd (b7f53597-227a-4284-97e3-4630087c3d27) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.128647352Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203222023-12-10T18:57:56.141Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.128668158Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203232023-12-10T18:57:56.141Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:56.128709865Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203242023-12-10T18:57:56.141Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.128856065Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203252023-12-10T18:57:56.141Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.128915973Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203262023-12-10T18:57:56.141Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.12893932Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203272023-12-10T18:57:56.141Z{"msg":"d2a74b43-bb99-4a51-8053-d9fa2b57c4cd WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.128960695Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203282023-12-10T18:57:56.141Z{"msg":"d2a74b43-bb99-4a51-8053-d9fa2b57c4cd WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.129002798Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203292023-12-10T18:57:56.142Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.129025403Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203302023-12-10T18:57:56.142Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.129045386Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203312023-12-10T18:57:56.142Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.129065424Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203322023-12-10T18:57:56.142Z{"msg":"d2a74b43-bb99-4a51-8053-d9fa2b57c4cd WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.129086197Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203332023-12-10T18:57:56.142Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.12913498Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203342023-12-10T18:57:56.142Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.129157028Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203352023-12-10T18:57:56.142Z{"msg":"[1] d2a74b43-bb99-4a51-8053-d9fa2b57c4cd (b7f53597-227a-4284-97e3-4630087c3d27) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.129299767Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203362023-12-10T18:57:56.142Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.129347758Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203372023-12-10T18:57:56.142Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:57:56.129494241Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203382023-12-10T18:57:56.143Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.129520737Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203392023-12-10T18:57:56.143Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.129542864Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203402023-12-10T18:57:56.143Z{"msg":"d2a74b43-bb99-4a51-8053-d9fa2b57c4cd WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.129585115Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203412023-12-10T18:57:56.143Z{"msg":"[0]R flush_numbers[0..12]: [288, 287, 282, 287, 285, 283, 287, 285, 275, 261, 280, 281]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.129609676Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203422023-12-10T18:57:56.143Z{"msg":"[0]R generation[0..12]: [165, 164, 161, 164, 163, 162, 164, 163, 158, 150, 159, 160]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.129631343Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203432023-12-10T18:57:56.143Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.12968279Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203442023-12-10T18:57:56.143Z{"The guest has finished waiting for activation with:166
203452023-12-10T18:57:56.143Zmsg":"[1]R flush_numbers[0..12]: [288, 287, 282, 287, 285, 283, 287, 285, 275, 261, 280, 281]","vWait for a query_work_queue command to finish before sending IO
203462023-12-10T18:57:56.143Z":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.129713442Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203472023-12-10T18:57:56.143Z{"msg":"[1]R generation[0..12]: [165, 164, 161, 164, 163, 162, 164, 163, 158, 150, 159, 160]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.129735928Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203482023-12-10T18:57:56.143Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.129757577Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203492023-12-10T18:57:56.144Z{"msg":"[2]R flush_numbers[0..12]: [288, 287, 282, 287, 285, 283, 287, 285, 275, 261, 280, 281]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.129779202Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203502023-12-10T18:57:56.144Z{"msg":"[2]R generation[0..12]: [165, 164, 161, 164, 163, 162, 164, 163, 158, 150, 159, 160]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.129848099Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203512023-12-10T18:57:56.144Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.129889752Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203522023-12-10T18:57:56.144Z{"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.129926397Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203532023-12-10T18:57:56.144Z{"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.129964481Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203542023-12-10T18:57:56.144Z{"msg":"Next flush: 289","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.130026384Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203552023-12-10T18:57:56.144Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.130065376Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203562023-12-10T18:57:56.145Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.130127947Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203572023-12-10T18:57:56.145Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.13016762Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203582023-12-10T18:57:56.145Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.130228801Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203592023-12-10T18:57:56.145Z{"msg":"d2a74b43-bb99-4a51-8053-d9fa2b57c4cd is now active with session: b7f53597-227a-4284-97e3-4630087c3d27","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.13027078Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203602023-12-10T18:57:56.145Z{"msg":"d2a74b43-bb99-4a51-8053-d9fa2b57c4cd Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.130378062Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203612023-12-10T18:57:56.145Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.130417699Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203622023-12-10T18:57:56.145Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.130456698Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203632023-12-10T18:57:56.145Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.130513525Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203642023-12-10T18:57:56.145Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.130537607Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203652023-12-10T18:57:56.145Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.130579223Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203662023-12-10T18:57:56.145Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.13060128Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203672023-12-10T18:57:56.145Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.130623304Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203682023-12-10T18:57:56.145Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.130643684Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203692023-12-10T18:57:56.145Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.13066293Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203702023-12-10T18:57:56.145Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.130682553Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203712023-12-10T18:57:56.146Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:57:56.130740983Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1826,"session_id":"b7f53597-227a-4284-97e3-4630087c3d27"}
203722023-12-10T18:57:56.146ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
203732023-12-10T18:57:56.146ZLoading write count information from file "/tmp/test_repair_verify.data"
203742023-12-10T18:57:56.146ZRun Repair workload
203752023-12-10T18:57:56.146Z01/30 Write block 409 len 32768 data: 13 13 10 9 8 9 9 10
203762023-12-10T18:57:56.146Z02/30 Write block 537 len 28672 data: 11 11 12 18 20 20 19
203772023-12-10T18:57:56.146Z03/30 Read block 7 len 16384
203782023-12-10T18:57:56.150Z04/30 Flush
203792023-12-10T18:57:56.150Z05/30 Write block 525 len 16384 data: 13 15 14 14
203802023-12-10T18:57:56.150Z06/30 Read block 288 len 28672
203812023-12-10T18:57:56.150Z07/30 Read block 481 len 16384
203822023-12-10T18:57:56.150Z08/30 Write block 3 len 24576 data: 5 4 5 6 10 13
203832023-12-10T18:57:56.150Z09/30 Write block 574 len 8192 data: 13 14
203842023-12-10T18:57:56.150Z10/30 Read block 31 len 16384
203852023-12-10T18:57:56.153Z11/30 Read block 192 len 8192
203862023-12-10T18:57:56.153Z12/30 Write block 365 len 36864 data: 7 8 8 8 9 8 11 12 11
203872023-12-10T18:57:56.158Z13/30 Write block 14 len 32768 data: 19 18 15 16 13 14 15 13
203882023-12-10T18:57:56.158Z14/30 Read block 178 len 8192
203892023-12-10T18:57:56.162Z15/30 Write block 487 len 28672 data: 14 14 14 12 14 9 8
203902023-12-10T18:57:56.162Z16/30 Read block 2 len 12288
203912023-12-10T18:57:56.162Z17/30 Read block 189 len 12288
203922023-12-10T18:57:56.162Z18/30 Read block 62 len 28672
203932023-12-10T18:57:56.165Z19/30 Flush
203942023-12-10T18:57:56.169Z20/30 Flush
203952023-12-10T18:57:56.170Z21/30 Write block 395 len 4096 data: 12
203962023-12-10T18:57:56.170Z22/30 Write block 177 len 24576 data: 11 9 8 7 9 12
203972023-12-10T18:57:56.173Z23/30 Flush
203982023-12-10T18:57:56.176Z24/30 Write block 50 len 28672 data: 10 9 8 7 9 8 9
203992023-12-10T18:57:56.176Z25/30 Write block 373 len 16384 data: 12 12 16 19
204002023-12-10T18:57:56.176Z26/30 Read block 312 len 40960
204012023-12-10T18:57:56.179Z27/30 Write block 240 len 24576 data: 18 18 19 17 18 18
204022023-12-10T18:57:56.182Z28/30 Read block 479 len 8192
204032023-12-10T18:57:56.182Z29/30 Write block 206 len 24576 data: 9 10 9 9 9 10
204042023-12-10T18:57:56.185Z30/30 Write block 373 len 12288 data: 13 13 17
204052023-12-10T18:57:56.185Z----------------------------------------------------------------
204062023-12-10T18:57:56.185Z Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:30
204072023-12-10T18:57:56.185ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
204082023-12-10T18:57:56.186Z 1 Acked 1000 Write 8 Done Sent Done false
204092023-12-10T18:57:56.186Z 2 Acked 1001 Write 7 Done Sent Done false
204102023-12-10T18:57:56.186Z 3 Acked 1002 Read 4 Done Sent Done false
204112023-12-10T18:57:56.186Z 4 Acked 1003 Flush 0 Done Sent Done false
204122023-12-10T18:57:56.186Z 5 Acked 1004 Write 4 Done Sent Done false
204132023-12-10T18:57:56.186Z 6 Acked 1005 Read 7 Done Sent Done false
204142023-12-10T18:57:56.186Z 7 Acked 1006 Read 4 Done Sent Done false
204152023-12-10T18:57:56.186Z 8 Acked 1007 Write 6 Done Sent Done false
204162023-12-10T18:57:56.186Z 9 Acked 1008 Write 2 Done Sent Done false
204172023-12-10T18:57:56.186Z 10 Acked 1009 Read 4 Done Sent Done false
204182023-12-10T18:57:56.187Z 11 Acked 1010 Read 2 Done Sent Done false
204192023-12-10T18:57:56.187Z 12 Acked 1011 Write 9 Done Sent Done false
204202023-12-10T18:57:56.187Z 13 Acked 1012 Write 8 Done Sent Done false
204212023-12-10T18:57:56.187Z 14 Acked 1013 Read 2 Done Sent Done false
204222023-12-10T18:57:56.187Z 15 Acked 1014 Write 7 Done Sent Done false
204232023-12-10T18:57:56.187Z 16 Acked 1015 Read 3 Done Sent Done false
204242023-12-10T18:57:56.187Z 17 Acked 1016 Read 3 Done Sent Done false
204252023-12-10T18:57:56.187Z 18 Acked 1017 Read 7 Done Sent Done false
204262023-12-10T18:57:56.187Z 19 Acked 1018 Flush 0 Done Sent Done false
204272023-12-10T18:57:56.187Z 20 Acked 1019 Flush 0 Done Sent Done false
204282023-12-10T18:57:56.187Z 21 Acked 1020 Write 1 Done Sent Done false
204292023-12-10T18:57:56.187Z 22 Acked 1021 Write 6 Done Sent Done false
204302023-12-10T18:57:56.187Z 23 Acked 1022 Flush 0 Done Sent Done false
204312023-12-10T18:57:56.187Z 24 Acked 1023 Write 7 Done Sent Done false
204322023-12-10T18:57:56.187Z 25 Acked 1024 Write 4 Done Sent Done false
204332023-12-10T18:57:56.187Z 26 Acked 1025 Read 10 Done Sent Done false
204342023-12-10T18:57:56.188Z 27 Acked 1026 Write 6 Done Sent Done false
204352023-12-10T18:57:56.188Z 28 Acked 1027 Read 2 Done Sent Done false
204362023-12-10T18:57:56.188Z 29 Acked 1028 Write 6 Done Sent Done false
204372023-12-10T18:57:56.188Z 30 Acked 1029 Write 3 Sent Sent Sent false
204382023-12-10T18:57:56.188Z STATES DS:0 DS:1 DS:2 TOTAL
204392023-12-10T18:57:56.188Z New 0 0 0 0
204402023-12-10T18:57:56.188Z Sent 1 30 1 32
204412023-12-10T18:57:56.188Z Done 29 0 29 58
204422023-12-10T18:57:56.189Z Skipped 0 0 0 0
204432023-12-10T18:57:56.189Z Error 0 0 0 0
204442023-12-10T18:57:56.189ZLast Flush: 1022 0 1022
204452023-12-10T18:57:56.189ZDownstairs last five completed:
204462023-12-10T18:57:56.189ZUpstairs last five completed: 30 29 28 27 26 25
204472023-12-10T18:57:56.189ZWrote out file "/tmp/test_repair_verify.data"
204482023-12-10T18:57:56.192Z
204492023-12-10T18:57:56.196ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
204502023-12-10T18:57:56.220ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
204512023-12-10T18:57:56.220Z 0 000-019 166 165 166 290 288 290 F F F <---
204522023-12-10T18:57:56.225Z 1 020-039 166 164 166 290 287 290 F F F <---
204532023-12-10T18:57:56.225Z 2 040-059 161 161 161 282 282 282 T F T <---
204542023-12-10T18:57:56.225Z 3 060-079 164 164 164 287 287 287 F F F
204552023-12-10T18:57:56.225Z 4 080-099 163 163 163 285 285 285 F F F
204562023-12-10T18:57:56.226Z 5 100-119 162 162 162 283 283 283 F F F
204572023-12-10T18:57:56.226Z 6 120-139 164 164 164 287 287 287 F F F
204582023-12-10T18:57:56.226Z 7 140-159 163 163 163 285 285 285 F F F
204592023-12-10T18:57:56.226Z 8 160-179 166 158 166 292 275 292 F F F <---
204602023-12-10T18:57:56.226Z 9 180-199 166 150 166 292 261 292 F F F <---
204612023-12-10T18:57:56.226Z 10 200-219 159 159 159 280 280 280 T F T <---
204622023-12-10T18:57:56.226Z 11 220-239 160 160 160 281 281 281 F F F
204632023-12-10T18:57:56.226Z 12 240-259 164 164 164 287 287 287 T F T <---
204642023-12-10T18:57:56.226Z 13 260-279 162 162 162 283 283 283 F F F
204652023-12-10T18:57:56.226Z 14 280-299 164 164 164 287 287 287 F F F
204662023-12-10T18:57:56.226Z 15 300-319 164 164 164 287 287 287 F F F
204672023-12-10T18:57:56.226Z 16 320-339 164 164 164 287 287 287 F F F
204682023-12-10T18:57:56.226Z 17 340-359 161 161 161 282 282 282 F F F
204692023-12-10T18:57:56.226Z 18 360-379 166 161 166 290 282 290 T F T <---
204702023-12-10T18:57:56.226Z 19 380-399 166 164 166 292 287 292 F F F <---
204712023-12-10T18:57:56.226Z 20 400-419 166 162 166 289 283 289 F F F <---
204722023-12-10T18:57:56.226Z 21 420-439 161 161 161 282 282 282 F F F
204732023-12-10T18:57:56.226Z 22 440-459 162 162 162 283 283 283 F F F
204742023-12-10T18:57:56.227Z 23 460-479 162 162 162 284 284 284 F F F
204752023-12-10T18:57:56.227Z 24 480-499 166 162 166 290 284 290 F F F <---
204762023-12-10T18:57:56.227Z 25 500-519 164 164 164 286 286 286 F F F
204772023-12-10T18:57:56.227Z 26 520-539 166 164 166 290 286 290 F F F <---
204782023-12-10T18:57:56.227Z 27 540-559 166 164 166 289 287 289 F F F <---
204792023-12-10T18:57:56.227Z 28 560-579 166 164 166 290 287 290 F F F <---
204802023-12-10T18:57:56.227Z 29 580-599 164 164 164 286 286 286 F F F
204812023-12-10T18:57:56.227ZMax gen: 166, Max flush: 292
204822023-12-10T18:57:56.227ZError: Difference in extent metadata found!
204832023-12-10T18:57:56.227ZA Difference in extent metadata is expected here
204842023-12-10T18:57:56.227ZOn loop 83
204852023-12-10T18:57:56.227Z
204862023-12-10T18:57:56.230ZVerifying data now
204872023-12-10T18:58:01.547ZNo change after verify
204882023-12-10T18:58:01.548ZLoop: 83 Downstairs dump after verify (and repair):
204892023-12-10T18:58:01.570ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
204902023-12-10T18:58:01.570Z 0 000-019 166 166 166 290 290 290 F F F
204912023-12-10T18:58:01.574Z 1 020-039 166 166 166 290 290 290 F F F
204922023-12-10T18:58:01.574Z 2 040-059 167 167 167 293 293 293 F F F
204932023-12-10T18:58:01.574Z 3 060-079 164 164 164 287 287 287 F F F
204942023-12-10T18:58:01.574Z 4 080-099 163 163 163 285 285 285 F F F
204952023-12-10T18:58:01.574Z 5 100-119 162 162 162 283 283 283 F F F
204962023-12-10T18:58:01.574Z 6 120-139 164 164 164 287 287 287 F F F
204972023-12-10T18:58:01.574Z 7 140-159 163 163 163 285 285 285 F F F
204982023-12-10T18:58:01.574Z 8 160-179 166 166 166 292 292 292 F F F
204992023-12-10T18:58:01.574Z 9 180-199 166 166 166 292 292 292 F F F
205002023-12-10T18:58:01.574Z 10 200-219 167 167 167 293 293 293 F F F
205012023-12-10T18:58:01.574Z 11 220-239 160 160 160 281 281 281 F F F
205022023-12-10T18:58:01.575Z 12 240-259 167 167 167 293 293 293 F F F
205032023-12-10T18:58:01.575Z 13 260-279 162 162 162 283 283 283 F F F
205042023-12-10T18:58:01.575Z 14 280-299 164 164 164 287 287 287 F F F
205052023-12-10T18:58:01.575Z 15 300-319 164 164 164 287 287 287 F F F
205062023-12-10T18:58:01.575Z 16 320-339 164 164 164 287 287 287 F F F
205072023-12-10T18:58:01.575Z 17 340-359 161 161 161 282 282 282 F F F
205082023-12-10T18:58:01.575Z 18 360-379 167 167 167 293 293 293 F F F
205092023-12-10T18:58:01.575Z 19 380-399 166 166 166 292 292 292 F F F
205102023-12-10T18:58:01.575Z 20 400-419 166 166 166 289 289 289 F F F
205112023-12-10T18:58:01.575Z 21 420-439 161 161 161 282 282 282 F F F
205122023-12-10T18:58:01.575Z 22 440-459 162 162 162 283 283 283 F F F
205132023-12-10T18:58:01.575Z 23 460-479 162 162 162 284 284 284 F F F
205142023-12-10T18:58:01.575Z 24 480-499 166 166 166 290 290 290 F F F
205152023-12-10T18:58:01.575Z 25 500-519 164 164 164 286 286 286 F F F
205162023-12-10T18:58:01.575Z 26 520-539 166 166 166 290 290 290 F F F
205172023-12-10T18:58:01.575Z 27 540-559 166 166 166 289 289 289 F F F
205182023-12-10T18:58:01.575Z 28 560-579 166 166 166 290 290 290 F F F
205192023-12-10T18:58:01.575Z 29 580-599 164 164 164 286 286 286 F F F
205202023-12-10T18:58:01.576ZMax gen: 167, Max flush: 293
205212023-12-10T18:58:01.579Z
205222023-12-10T18:58:01.579ZBegin loop 84
205232023-12-10T18:58:01.580ZDownstairs to restart: 1
205242023-12-10T18:58:01.605Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.603029664Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834}
205252023-12-10T18:58:01.605Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.603460892Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834}
205262023-12-10T18:58:01.610Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.60349557Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834}
205272023-12-10T18:58:01.610Z{"msg":"Crucible 39ec1a32-a837-437e-94f2-d6933ee9c558 has session id: 1d170eff-f43f-4a7c-b576-3e5df8beeec4","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.603564942Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205282023-12-10T18:58:01.610Z{"msg":"Upstairs opts: Upstairs UUID: 39ec1a32-a837-437e-94f2-d6933ee9c558, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.603626352Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205292023-12-10T18:58:01.611Z{"msg":"Crucible stats registered with UUID: 39ec1a32-a837-437e-94f2-d6933ee9c558","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.603739026Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205302023-12-10T18:58:01.611Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30Crucible runtime is spawned
205312023-12-10T18:58:01.611Z,"time":"2023-12-10T18:58:01.603904502Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"looper":"0","session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205322023-12-10T18:58:01.611Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30The guest has requested activation with gen:168
205332023-12-10T18:58:01.611Z,"time":"2023-12-10T18:58:01.60412691Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"looper":"1","session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205342023-12-10T18:58:01.611Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.604445687Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"looper":"2","session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205352023-12-10T18:58:01.611Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.604471302Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4","task":"up_listen"}
205362023-12-10T18:58:01.611Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.604589628Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205372023-12-10T18:58:01.611Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.604614393Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205382023-12-10T18:58:01.611Z{"msg":"39ec1a32-a837-437e-94f2-d6933ee9c558 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.604670287Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205392023-12-10T18:58:01.611Z{"msg":"[2] 39ec1a32-a837-437e-94f2-d6933ee9c558 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.604694868Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"looper":"2","session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205402023-12-10T18:58:01.611Z{"msg":"Set desired generation to :168","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.604718058Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205412023-12-10T18:58:01.611Z{"msg":"[0] 39ec1a32-a837-437e-94f2-d6933ee9c558 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.604738772Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"looper":"0","session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205422023-12-10T18:58:01.611Z{"msg":"[1] 39ec1a32-a837-437e-94f2-d6933ee9c558 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.604800797Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"looper":"1","session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205432023-12-10T18:58:01.611Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.604826159Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205442023-12-10T18:58:01.611Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.604852822Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205452023-12-10T18:58:01.612Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.605293482Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205462023-12-10T18:58:01.614Z{"msg":"[0] 39ec1a32-a837-437e-94f2-d6933ee9c558 (1d170eff-f43f-4a7c-b576-3e5df8beeec4) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.605524573Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205472023-12-10T18:58:01.614Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.605585149Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205482023-12-10T18:58:01.614Z{"msg":"[0] client is_active_req TRUE, promote! session 1d170eff-f43f-4a7c-b576-3e5df8beeec4","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.605611314Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205492023-12-10T18:58:01.614Z{"msg":"[2] 39ec1a32-a837-437e-94f2-d6933ee9c558 (1d170eff-f43f-4a7c-b576-3e5df8beeec4) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.6056345Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205502023-12-10T18:58:01.614Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.605684299Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205512023-12-10T18:58:01.614Z{"msg":"[2] client is_active_req TRUE, promote! session 1d170eff-f43f-4a7c-b576-3e5df8beeec4","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.605711748Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205522023-12-10T18:58:01.614Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.606281016Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205532023-12-10T18:58:01.614Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.606355995Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205542023-12-10T18:58:01.614Z{"msg":"39ec1a32-a837-437e-94f2-d6933ee9c558 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.606472067Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205552023-12-10T18:58:01.614Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.606556584Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205562023-12-10T18:58:01.615Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.606636527Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205572023-12-10T18:58:01.615Z{"msg":"39ec1a32-a837-437e-94f2-d6933ee9c558 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.606728794Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205582023-12-10T18:58:01.621Z{"msg":"[0] 39ec1a32-a837-437e-94f2-d6933ee9c558 (1d170eff-f43f-4a7c-b576-3e5df8beeec4) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.60740695Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205592023-12-10T18:58:01.622Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.607492678Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205602023-12-10T18:58:01.622Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:01.607523304Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205612023-12-10T18:58:01.622Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.607545543Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205622023-12-10T18:58:01.622Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.607566724Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205632023-12-10T18:58:01.622Z{"msg":"39ec1a32-a837-437e-94f2-d6933ee9c558 WaitQuorum New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.60758776Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205642023-12-10T18:58:01.622Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.607608257Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205652023-12-10T18:58:01.622Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.607629561Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205662023-12-10T18:58:01.622Z{"msg":"[2] 39ec1a32-a837-437e-94f2-d6933ee9c558 (1d170eff-f43f-4a7c-b576-3e5df8beeec4) WaitQuorum New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.607671702Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205672023-12-10T18:58:01.623Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.607747057Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205682023-12-10T18:58:01.623Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:01.607770353Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205692023-12-10T18:58:01.623Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.607790326Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205702023-12-10T18:58:01.623Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.607837539Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205712023-12-10T18:58:01.623Z{"msg":"[1] 39ec1a32-a837-437e-94f2-d6933ee9c558 (1d170eff-f43f-4a7c-b576-3e5df8beeec4) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.607863476Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205722023-12-10T18:58:01.623Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.607885588Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205732023-12-10T18:58:01.623Z{"msg":"[1] client is_active_req TRUE, promote! session 1d170eff-f43f-4a7c-b576-3e5df8beeec4","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.607908766Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205742023-12-10T18:58:01.623Z{"msg":"39ec1a32-a837-437e-94f2-d6933ee9c558 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.608032819Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205752023-12-10T18:58:01.623Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.608083342Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205762023-12-10T18:58:01.623Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.608106597Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205772023-12-10T18:58:01.623Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.608700971Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205782023-12-10T18:58:01.623Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.608761469Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205792023-12-10T18:58:01.623Z{"msg":"39ec1a32-a837-437e-94f2-d6933ee9c558 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.608786486Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205802023-12-10T18:58:01.623Z{"msg":"[1] 39ec1a32-a837-437e-94f2-d6933ee9c558 (1d170eff-f43f-4a7c-b576-3e5df8beeec4) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.609421303Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205812023-12-10T18:58:01.623Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.60949599Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205822023-12-10T18:58:01.623Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:01.609534154Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205832023-12-10T18:58:01.624Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.609579676Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205842023-12-10T18:58:01.624Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.609602806Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205852023-12-10T18:58:01.624Z{"msg":"39ec1a32-a837-437e-94f2-d6933ee9c558 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.609643756Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205862023-12-10T18:58:01.624Z{"msg":"[0]R flush_numbers[0..12]: [290, 290, 293, 287, 285, 283, 287, 285, 292, 292, 293, 281]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.609667631Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205872023-12-10T18:58:01.624Z{"msg":"[0]R generation[0..12]: [166, 166, 167, 164, 163, 162, 164, 163, 166, 166, 167, 160]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.609689406Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205882023-12-10T18:58:01.624Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.609730934Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205892023-12-10T18:58:01.624Z{"msg":"[1]R flush_numbers[0..12]: [290, 290, 293, 287, 285, 283, 287, 285, 292, 292, 293, 281]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.609754392Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205902023-12-10T18:58:01.624Z{"msg":"[1]R generation[0..12]: [166, 166, 167, 164, 163, 162, 164, 163, 166, 166, 167, 160]","v":0,"name":"crucible","level"The guest has finished waiting for activation with:168
205912023-12-10T18:58:01.624ZWait for a query_work_queue command to finish before sending IO
205922023-12-10T18:58:01.624Z:30,"time":"2023-12-10T18:58:01.609836625Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205932023-12-10T18:58:01.624Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.60989001Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205942023-12-10T18:58:01.624Z{"msg":"[2]R flush_numbers[0..12]: [290, 290, 293, 287, 285, 283, 287, 285, 292, 292, 293, 281]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.609915202Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205952023-12-10T18:58:01.624Z{"msg":"[2]R generation[0..12]: [166, 166, 167, 164, 163, 162, 164, 163, 166, 166, 167, 160]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.609981689Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205962023-12-10T18:58:01.624Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.610006201Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205972023-12-10T18:58:01.624Z{"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.610026658Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205982023-12-10T18:58:01.624Z{"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.610089308Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
205992023-12-10T18:58:01.625Z{"msg":"Next flush: 294","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.610121554Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
206002023-12-10T18:58:01.625Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.610142987Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
206012023-12-10T18:58:01.625Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.610194617Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
206022023-12-10T18:58:01.625Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.61021734Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
206032023-12-10T18:58:01.625Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.610237566Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
206042023-12-10T18:58:01.625Z{"msg":"39ec1a32-a837-437e-94f2-d6933ee9c558 is now active with session: 1d170eff-f43f-4a7c-b576-3e5df8beeec4","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.610279534Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
206052023-12-10T18:58:01.625Z{"msg":"39ec1a32-a837-437e-94f2-d6933ee9c558 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.610310877Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
206062023-12-10T18:58:01.625Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.610353968Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
206072023-12-10T18:58:01.625Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.610376069Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
206082023-12-10T18:58:01.626Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.61039599Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
206092023-12-10T18:58:01.626Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.61044948Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
206102023-12-10T18:58:01.626Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.610479245Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
206112023-12-10T18:58:01.626Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.610499684Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
206122023-12-10T18:58:01.626Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.61051938Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
206132023-12-10T18:58:01.626Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.610594939Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
206142023-12-10T18:58:01.626Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.610662124Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
206152023-12-10T18:58:01.626Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.610751926Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
206162023-12-10T18:58:01.626Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:01.610819102Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1834,"session_id":"1d170eff-f43f-4a7c-b576-3e5df8beeec4"}
206172023-12-10T18:58:01.626ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
206182023-12-10T18:58:01.626ZLoading write count information from file "/tmp/test_repair_verify.data"
206192023-12-10T18:58:01.626ZRun Repair workload
206202023-12-10T18:58:01.626Z01/30 Write block 222 len 32768 data: 17 15 15 13 17 16 15 15
206212023-12-10T18:58:01.627Z02/30 Write block 252 len 4096 data: 15
206222023-12-10T18:58:01.627Z03/30 Read block 13 len 36864
206232023-12-10T18:58:01.630Z04/30 Read block 185 len 8192
206242023-12-10T18:58:01.630Z05/30 Write block 196 len 32768 data: 16 18 16 15 14 17 16 17
206252023-12-10T18:58:01.630Z06/30 Read block 476 len 16384
206262023-12-10T18:58:01.630Z07/30 Flush
206272023-12-10T18:58:01.631Z08/30 Write block 296 len 36864 data: 14 14 14 16 12 15 15 17 18
206282023-12-10T18:58:01.631Z09/30 Flush
206292023-12-10T18:58:01.634Z10/30 Write block 402 len 16384 data: 15 15 12 15
206302023-12-10T18:58:01.634Z11/30 Read block 5 len 36864
206312023-12-10T18:58:01.637Z12/30 Read block 369 len 40960
206322023-12-10T18:58:01.641Z13/30 Read block 556 len 32768
206332023-12-10T18:58:01.645Z14/30 Read block 383 len 16384
206342023-12-10T18:58:01.645Z15/30 Write block 399 len 28672 data: 13 14 15 16 16 13 16
206352023-12-10T18:58:01.645Z16/30 Read block 408 len 16384
206362023-12-10T18:58:01.651Z17/30 Read block 326 len 4096
206372023-12-10T18:58:01.651Z18/30 Write block 71 len 24576 data: 16 13 16 17 15 14
206382023-12-10T18:58:01.655Z19/30 Write block 352 len 24576 data: 18 15 14 13 15 13
206392023-12-10T18:58:01.655Z20/30 Write block 254 len 20480 data: 13 11 11 9 8
206402023-12-10T18:58:01.655Z21/30 Read block 90 len 4096
206412023-12-10T18:58:01.656Z22/30 Read block 271 len 20480
206422023-12-10T18:58:01.656Z23/30 Read block 11 len 28672
206432023-12-10T18:58:01.659Z24/30 Read block 564 len 20480
206442023-12-10T18:58:01.662Z25/30 Read block 190 len 4096
206452023-12-10T18:58:01.662Z26/30 Read block 522 len 12288
206462023-12-10T18:58:01.665Z27/30 Write block 277 len 12288 data: 17 15 12
206472023-12-10T18:58:01.666Z28/30 Read block 277 len 4096
206482023-12-10T18:58:01.666Z29/30 Write block 58 len 32768 data: 9 7 9 10 12 13 13 16
206492023-12-10T18:58:01.666Z30/30 Read block 498 len 24576
206502023-12-10T18:58:01.670Z----------------------------------------------------------------
206512023-12-10T18:58:01.670Z Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:30
206522023-12-10T18:58:01.670ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
206532023-12-10T18:58:01.670Z 1 Acked 1000 Write 8 Done Sent Done false
206542023-12-10T18:58:01.670Z 2 Acked 1001 Write 1 Done Sent Done false
206552023-12-10T18:58:01.670Z 3 Acked 1002 Read 9 Done Sent Done false
206562023-12-10T18:58:01.670Z 4 Acked 1003 Read 2 Done Sent Done false
206572023-12-10T18:58:01.670Z 5 Acked 1004 Write 8 Done Sent Done false
206582023-12-10T18:58:01.671Z 6 Acked 1005 Read 4 Done Sent Done false
206592023-12-10T18:58:01.671Z 7 Acked 1006 Flush 0 Done Sent Done false
206602023-12-10T18:58:01.671Z 8 Acked 1007 Write 9 Done Sent Done false
206612023-12-10T18:58:01.671Z 9 Acked 1008 Flush 0 Done Sent Done false
206622023-12-10T18:58:01.671Z 10 Acked 1009 Write 4 Done Sent Done false
206632023-12-10T18:58:01.671Z 11 Acked 1010 Read 9 Done Sent Done false
206642023-12-10T18:58:01.671Z 12 Acked 1011 Read 10 Done Sent Done false
206652023-12-10T18:58:01.671Z 13 Acked 1012 Read 8 Done Sent Done false
206662023-12-10T18:58:01.671Z 14 Acked 1013 Read 4 Done Sent Done false
206672023-12-10T18:58:01.671Z 15 Acked 1014 Write 7 Done Sent Done false
206682023-12-10T18:58:01.671Z 16 Acked 1015 Read 4 Done Sent Done false
206692023-12-10T18:58:01.671Z 17 Acked 1016 Read 1 Done Sent Done false
206702023-12-10T18:58:01.671Z 18 Acked 1017 Write 6 Done Sent Done false
206712023-12-10T18:58:01.671Z 19 Acked 1018 Write 6 Done Sent Done false
206722023-12-10T18:58:01.671Z 20 Acked 1019 Write 5 Done Sent Done false
206732023-12-10T18:58:01.671Z 21 Acked 1020 Read 1 Done Sent Done false
206742023-12-10T18:58:01.671Z 22 Acked 1021 Read 5 Done Sent Done false
206752023-12-10T18:58:01.672Z 23 Acked 1022 Read 7 Done Sent Done false
206762023-12-10T18:58:01.672Z 24 Acked 1023 Read 5 Done Sent Done false
206772023-12-10T18:58:01.672Z 25 Acked 1024 Read 1 Done Sent Done false
206782023-12-10T18:58:01.672Z 26 Acked 1025 Read 3 Done Sent Done false
206792023-12-10T18:58:01.672Z 27 Acked 1026 Write 3 Done Sent Done false
206802023-12-10T18:58:01.672Z 28 Acked 1027 Read 1 Done Sent Done false
206812023-12-10T18:58:01.672Z 29 Acked 1028 Write 8 Done Sent Done false
206822023-12-10T18:58:01.672Z 30 Acked 1029 Read 6 Done Sent Done false
206832023-12-10T18:58:01.672Z STATES DS:0 DS:1 DS:2 TOTAL
206842023-12-10T18:58:01.672Z New 0 0 0 0
206852023-12-10T18:58:01.672Z Sent 0 30 0 30
206862023-12-10T18:58:01.672Z Done 30 0 30 60
206872023-12-10T18:58:01.672Z Skipped 0 0 0 0
206882023-12-10T18:58:01.672Z Error 0 0 0 0
206892023-12-10T18:58:01.672ZLast Flush: 1008 0 1008
206902023-12-10T18:58:01.672ZDownstairs last five completed:
206912023-12-10T18:58:01.672ZUpstairs last five completed: 30 29 28 27 26 25
206922023-12-10T18:58:01.672ZWrote out file "/tmp/test_repair_verify.data"
206932023-12-10T18:58:01.677Z
206942023-12-10T18:58:01.682ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
206952023-12-10T18:58:01.711ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
206962023-12-10T18:58:01.711Z 0 000-019 166 166 166 290 290 290 F F F
206972023-12-10T18:58:01.711Z 1 020-039 166 166 166 290 290 290 F F F
206982023-12-10T18:58:01.711Z 2 040-059 167 167 167 293 293 293 T F T <---
206992023-12-10T18:58:01.711Z 3 060-079 164 164 164 287 287 287 T F T <---
207002023-12-10T18:58:01.711Z 4 080-099 163 163 163 285 285 285 F F F
207012023-12-10T18:58:01.711Z 5 100-119 162 162 162 283 283 283 F F F
207022023-12-10T18:58:01.711Z 6 120-139 164 164 164 287 287 287 F F F
207032023-12-10T18:58:01.711Z 7 140-159 163 163 163 285 285 285 F F F
207042023-12-10T18:58:01.716Z 8 160-179 166 166 166 292 292 292 F F F
207052023-12-10T18:58:01.716Z 9 180-199 168 166 168 294 292 294 F F F <---
207062023-12-10T18:58:01.716Z 10 200-219 168 167 168 294 293 294 F F F <---
207072023-12-10T18:58:01.716Z 11 220-239 168 160 168 294 281 294 F F F <---
207082023-12-10T18:58:01.716Z 12 240-259 168 167 168 294 293 294 T F T <---
207092023-12-10T18:58:01.716Z 13 260-279 162 162 162 283 283 283 T F T <---
207102023-12-10T18:58:01.716Z 14 280-299 168 164 168 295 287 295 F F F <---
207112023-12-10T18:58:01.716Z 15 300-319 168 164 168 295 287 295 F F F <---
207122023-12-10T18:58:01.716Z 16 320-339 164 164 164 287 287 287 F F F
207132023-12-10T18:58:01.716Z 17 340-359 161 161 161 282 282 282 T F T <---
207142023-12-10T18:58:01.716Z 18 360-379 167 167 167 293 293 293 F F F
207152023-12-10T18:58:01.716Z 19 380-399 166 166 166 292 292 292 T F T <---
207162023-12-10T18:58:01.716Z 20 400-419 166 166 166 289 289 289 T F T <---
207172023-12-10T18:58:01.717Z 21 420-439 161 161 161 282 282 282 F F F
207182023-12-10T18:58:01.719Z 22 440-459 162 162 162 283 283 283 F F F
207192023-12-10T18:58:01.719Z 23 460-479 162 162 162 284 284 284 F F F
207202023-12-10T18:58:01.719Z 24 480-499 166 166 166 290 290 290 F F F
207212023-12-10T18:58:01.719Z 25 500-519 164 164 164 286 286 286 F F F
207222023-12-10T18:58:01.719Z 26 520-539 166 166 166 290 290 290 F F F
207232023-12-10T18:58:01.719Z 27 540-559 166 166 166 289 289 289 F F F
207242023-12-10T18:58:01.719Z 28 560-579 166 166 166 290 290 290 F F F
207252023-12-10T18:58:01.719Z 29 580-599 164 164 164 286 286 286 F F F
207262023-12-10T18:58:01.719ZMax gen: 168, Max flush: 295
207272023-12-10T18:58:01.719ZError: Difference in extent metadata found!
207282023-12-10T18:58:01.719ZA Difference in extent metadata is expected here
207292023-12-10T18:58:01.720ZOn loop 84
207302023-12-10T18:58:01.720Z
207312023-12-10T18:58:01.723ZVerifying data now
207322023-12-10T18:58:06.944ZNo change after verify
207332023-12-10T18:58:06.944ZLoop: 84 Downstairs dump after verify (and repair):
207342023-12-10T18:58:06.966ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
207352023-12-10T18:58:06.966Z 0 000-019 166 166 166 290 290 290 F F F
207362023-12-10T18:58:06.966Z 1 020-039 166 166 166 290 290 290 F F F
207372023-12-10T18:58:06.966Z 2 040-059 169 169 169 296 296 296 F F F
207382023-12-10T18:58:06.966Z 3 060-079 169 169 169 296 296 296 F F F
207392023-12-10T18:58:06.966Z 4 080-099 163 163 163 285 285 285 F F F
207402023-12-10T18:58:06.971Z 5 100-119 162 162 162 283 283 283 F F F
207412023-12-10T18:58:06.971Z 6 120-139 164 164 164 287 287 287 F F F
207422023-12-10T18:58:06.971Z 7 140-159 163 163 163 285 285 285 F F F
207432023-12-10T18:58:06.971Z 8 160-179 166 166 166 292 292 292 F F F
207442023-12-10T18:58:06.971Z 9 180-199 168 168 168 294 294 294 F F F
207452023-12-10T18:58:06.971Z 10 200-219 168 168 168 294 294 294 F F F
207462023-12-10T18:58:06.971Z 11 220-239 168 168 168 294 294 294 F F F
207472023-12-10T18:58:06.971Z 12 240-259 169 169 169 296 296 296 F F F
207482023-12-10T18:58:06.972Z 13 260-279 169 169 169 296 296 296 F F F
207492023-12-10T18:58:06.972Z 14 280-299 168 168 168 295 295 295 F F F
207502023-12-10T18:58:06.972Z 15 300-319 168 168 168 295 295 295 F F F
207512023-12-10T18:58:06.972Z 16 320-339 164 164 164 287 287 287 F F F
207522023-12-10T18:58:06.972Z 17 340-359 169 169 169 296 296 296 F F F
207532023-12-10T18:58:06.972Z 18 360-379 167 167 167 293 293 293 F F F
207542023-12-10T18:58:06.972Z 19 380-399 169 169 169 296 296 296 F F F
207552023-12-10T18:58:06.972Z 20 400-419 169 169 169 296 296 296 F F F
207562023-12-10T18:58:06.972Z 21 420-439 161 161 161 282 282 282 F F F
207572023-12-10T18:58:06.972Z 22 440-459 162 162 162 283 283 283 F F F
207582023-12-10T18:58:06.972Z 23 460-479 162 162 162 284 284 284 F F F
207592023-12-10T18:58:06.972Z 24 480-499 166 166 166 290 290 290 F F F
207602023-12-10T18:58:06.972Z 25 500-519 164 164 164 286 286 286 F F F
207612023-12-10T18:58:06.972Z 26 520-539 166 166 166 290 290 290 F F F
207622023-12-10T18:58:06.972Z 27 540-559 166 166 166 289 289 289 F F F
207632023-12-10T18:58:06.972Z 28 560-579 166 166 166 290 290 290 F F F
207642023-12-10T18:58:06.973Z 29 580-599 164 164 164 286 286 286 F F F
207652023-12-10T18:58:06.973ZMax gen: 169, Max flush: 296
207662023-12-10T18:58:06.973Z
207672023-12-10T18:58:06.973ZBegin loop 85
207682023-12-10T18:58:06.973ZDownstairs to restart: 0
207692023-12-10T18:58:07.005Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.003131096Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842}
207702023-12-10T18:58:07.005Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.003529866Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842}
207712023-12-10T18:58:07.010Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.003597665Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842}
207722023-12-10T18:58:07.010Z{"msg":"Crucible d9b2d47c-d6c9-4000-8f08-843f20d01f01 has session id: 1675aa21-8ed9-4818-9e3b-9447101906a9","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.003668264Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
207732023-12-10T18:58:07.010Z{"msg":"Upstairs opts: Upstairs UUID: d9b2d47c-d6c9-4000-8f08-843f20d01f01, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.003732938Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
207742023-12-10T18:58:07.010Z{"msg":"Crucible stats registered with UUID: d9b2d47c-d6c9-4000-8f08-843f20d01f01","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.003813854Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
207752023-12-10T18:58:07.010ZCrucible runtime is spawned
207762023-12-10T18:58:07.010ZThe guest has requested activation with gen:170
207772023-12-10T18:58:07.010Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.004105773Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9","task":"up_listen"}
207782023-12-10T18:58:07.011Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.004228287Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"looper":"0","session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
207792023-12-10T18:58:07.011Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.004264304Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
207802023-12-10T18:58:07.011Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.004314908Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
207812023-12-10T18:58:07.011Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.004469508Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"looper":"2","session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
207822023-12-10T18:58:07.011Z{"msg":"[0] d9b2d47c-d6c9-4000-8f08-843f20d01f01 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.00455491Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"looper":"0","session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
207832023-12-10T18:58:07.011Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.004587283Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"looper":"1","session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
207842023-12-10T18:58:07.011Z{"msg":"[2] d9b2d47c-d6c9-4000-8f08-843f20d01f01 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.004715259Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"looper":"2","session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
207852023-12-10T18:58:07.011Z{"msg":"[1] d9b2d47c-d6c9-4000-8f08-843f20d01f01 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.004819847Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"looper":"1","session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
207862023-12-10T18:58:07.012Z{"msg":"d9b2d47c-d6c9-4000-8f08-843f20d01f01 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.005285146Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
207872023-12-10T18:58:07.012Z{"msg":"Set desired generation to :170","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.005472755Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
207882023-12-10T18:58:07.012Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.005553893Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
207892023-12-10T18:58:07.012Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.0058177Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
207902023-12-10T18:58:07.012Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.005926275Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
207912023-12-10T18:58:07.012Z{"msg":"[1] d9b2d47c-d6c9-4000-8f08-843f20d01f01 (1675aa21-8ed9-4818-9e3b-9447101906a9) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.006438064Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
207922023-12-10T18:58:07.012Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.00651556Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
207932023-12-10T18:58:07.012Z{"msg":"[1] client is_active_req TRUE, promote! session 1675aa21-8ed9-4818-9e3b-9447101906a9","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.006602872Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
207942023-12-10T18:58:07.012Z{"msg":"[0] d9b2d47c-d6c9-4000-8f08-843f20d01f01 (1675aa21-8ed9-4818-9e3b-9447101906a9) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.006692835Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
207952023-12-10T18:58:07.012Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.006734625Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
207962023-12-10T18:58:07.012Z{"msg":"[0] client is_active_req TRUE, promote! session 1675aa21-8ed9-4818-9e3b-9447101906a9","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.006855068Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
207972023-12-10T18:58:07.012Z{"msg":"[2] d9b2d47c-d6c9-4000-8f08-843f20d01f01 (1675aa21-8ed9-4818-9e3b-9447101906a9) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.006881796Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
207982023-12-10T18:58:07.012Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.006908557Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
207992023-12-10T18:58:07.013Z{"msg":"[2] client is_active_req TRUE, promote! session 1675aa21-8ed9-4818-9e3b-9447101906a9","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.0069702Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208002023-12-10T18:58:07.022Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.0072548Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208012023-12-10T18:58:07.022Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.007357115Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208022023-12-10T18:58:07.022Z{"msg":"d9b2d47c-d6c9-4000-8f08-843f20d01f01 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.007431258Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208032023-12-10T18:58:07.022Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.007461385Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208042023-12-10T18:58:07.022Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.007494189Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208052023-12-10T18:58:07.022Z{"msg":"d9b2d47c-d6c9-4000-8f08-843f20d01f01 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.007566185Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208062023-12-10T18:58:07.022Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.007613426Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208072023-12-10T18:58:07.022Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.007693257Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208082023-12-10T18:58:07.023Z{"msg":"d9b2d47c-d6c9-4000-8f08-843f20d01f01 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.007766957Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208092023-12-10T18:58:07.023Z{"msg":"[1] d9b2d47c-d6c9-4000-8f08-843f20d01f01 (1675aa21-8ed9-4818-9e3b-9447101906a9) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.008164998Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208102023-12-10T18:58:07.023Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.008279436Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208112023-12-10T18:58:07.023Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:07.008342097Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208122023-12-10T18:58:07.023Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.008367223Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208132023-12-10T18:58:07.023Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.008388215Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208142023-12-10T18:58:07.023Z{"msg":"d9b2d47c-d6c9-4000-8f08-843f20d01f01 WaitActive WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.008409459Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208152023-12-10T18:58:07.023Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.008430182Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208162023-12-10T18:58:07.024Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.008452918Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208172023-12-10T18:58:07.024Z{"msg":"[2] d9b2d47c-d6c9-4000-8f08-843f20d01f01 (1675aa21-8ed9-4818-9e3b-9447101906a9) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.008606384Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208182023-12-10T18:58:07.024Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.008674628Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208192023-12-10T18:58:07.024Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:07.008699311Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208202023-12-10T18:58:07.024Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.008720081Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208212023-12-10T18:58:07.024Z{"msg":"[0] d9b2d47c-d6c9-4000-8f08-843f20d01f01 (1675aa21-8ed9-4818-9e3b-9447101906a9) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.00874335Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208222023-12-10T18:58:07.024Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.008764364Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208232023-12-10T18:58:07.024Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:07.008784543Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208242023-12-10T18:58:07.025Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.008817521Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208252023-12-10T18:58:07.025Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.0088421Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208262023-12-10T18:58:07.025Z{"msg":"d9b2d47c-d6c9-4000-8f08-843f20d01f01 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.008886504Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208272023-12-10T18:58:07.025Z{"msg":"[0]R flush_numbers[0..12]: [290, 290, 296, 296, 285, 283, 287, 285, 292, 294, 294, 294]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.008911805Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208282023-12-10T18:58:07.025Z{"msg":"[0]R generation[0..12]: [166, 166, 169, 169, 163, 162, 164, 163, 166, 168, 168, 168]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.008933644Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208292023-12-10T18:58:07.025Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.008976423Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208302023-12-10T18:58:07.026Z{"msg":"[1]R flush_numbers[0..12]: [290, 290, 296, 296, 285, 283, 287, 285, 292, 294, 294, 294]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.009001258Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208312023-12-10T18:58:07.026Z{"msg":"[1]R generation[0..12]: [166, 166, 169, 169, 163, 162, 164, 163, 166, 168, 168, 168]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.009023159Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208322023-12-10T18:58:07.026Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.009068899Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208332023-12-10T18:58:07.026Z{"msg":"[2]R flush_numbers[0..12]: [290, 290, 296, 296, 285, 283, 287, 285, 292, 294, 294, 294]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.009093221Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208342023-12-10T18:58:07.026Z{"msg":"[2]R generation[0..12]: [166, 166, 169, 169, 163, 162, 164, 163, 166, 168, 168, 168]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:170
208352023-12-10T18:58:07.026Z,"time":"Wait for a query_work_queue command to finish before sending IO
208362023-12-10T18:58:07.026Z2023-12-10T18:58:07.009114848Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208372023-12-10T18:58:07.026Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.009139383Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208382023-12-10T18:58:07.026Z{"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.009188993Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208392023-12-10T18:58:07.027Z{"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.009212068Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208402023-12-10T18:58:07.027Z{"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.009231861Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208412023-12-10T18:58:07.027Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.009286434Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208422023-12-10T18:58:07.027Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.009309833Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208432023-12-10T18:58:07.027Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.00933062Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208442023-12-10T18:58:07.027Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.009391591Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208452023-12-10T18:58:07.027Z{"msg":"d9b2d47c-d6c9-4000-8f08-843f20d01f01 is now active with session: 1675aa21-8ed9-4818-9e3b-9447101906a9","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.009419159Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208462023-12-10T18:58:07.027Z{"msg":"d9b2d47c-d6c9-4000-8f08-843f20d01f01 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.009441556Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208472023-12-10T18:58:07.027Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.009501887Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208482023-12-10T18:58:07.027Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.009525146Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208492023-12-10T18:58:07.027Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.009548567Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208502023-12-10T18:58:07.028Z{"msg":"d9b2d47c-d6c9-4000-8f08-843f20d01f01 Active Active Active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.009625467Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208512023-12-10T18:58:07.028Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.009653608Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208522023-12-10T18:58:07.028Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.009697516Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208532023-12-10T18:58:07.028Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.009720751Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208542023-12-10T18:58:07.028Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.009740487Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208552023-12-10T18:58:07.028Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.009783072Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208562023-12-10T18:58:07.028Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.00980583Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208572023-12-10T18:58:07.028Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.009846582Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208582023-12-10T18:58:07.029Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.00986913Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208592023-12-10T18:58:07.029Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.00989282Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208602023-12-10T18:58:07.029Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:07.009959385Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1842,"session_id":"1675aa21-8ed9-4818-9e3b-9447101906a9"}
208612023-12-10T18:58:07.029ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
208622023-12-10T18:58:07.029ZLoading write count information from file "/tmp/test_repair_verify.data"
208632023-12-10T18:58:07.030ZRun Repair workload
208642023-12-10T18:58:07.030Z01/30 Write block 408 len 8192 data: 16 14
208652023-12-10T18:58:07.030Z02/30 Write block 559 len 32768 data: 13 12 15 17 14 14 14 15
208662023-12-10T18:58:07.030Z03/30 Read block 328 len 32768
208672023-12-10T18:58:07.033Z04/30 Write block 234 len 36864 data: 19 19 16 15 14 18 19 19 20
208682023-12-10T18:58:07.033Z05/30 Read block 165 len 20480
208692023-12-10T18:58:07.034Z06/30 Flush
208702023-12-10T18:58:07.034Z07/30 Write block 314 len 40960 data: 12 12 11 10 13 14 13 13 13 13
208712023-12-10T18:58:07.034Z08/30 Read block 136 len 40960
208722023-12-10T18:58:07.034Z09/30 Read block 16 len 20480
208732023-12-10T18:58:07.034Z10/30 Read block 536 len 28672
208742023-12-10T18:58:07.037Z11/30 Write block 530 len 24576 data: 13 17 17 15 17 16
208752023-12-10T18:58:07.037Z12/30 Write block 355 len 36864 data: 14 16 14 13 15 13 12 11 11
208762023-12-10T18:58:07.040Z13/30 Write block 420 len 28672 data: 12 14 15 15 17 14 14
208772023-12-10T18:58:07.040Z14/30 Read block 537 len 12288
208782023-12-10T18:58:07.043Z15/30 Write block 61 len 20480 data: 11 13 14 14 17
208792023-12-10T18:58:07.043Z16/30 Flush
208802023-12-10T18:58:07.047Z17/30 Write block 476 len 40960 data: 9 7 5 6 7 7 9 10 10 11
208812023-12-10T18:58:07.050Z18/30 Read block 341 len 36864
208822023-12-10T18:58:07.053Z19/30 Read block 422 len 4096
208832023-12-10T18:58:07.053Z20/30 Read block 16 len 20480
208842023-12-10T18:58:07.056Z21/30 Read block 247 len 24576
208852023-12-10T18:58:07.059Z22/30 Write block 295 len 24576 data: 10 15 15 15 17 13
208862023-12-10T18:58:07.059Z23/30 Read block 445 len 32768
208872023-12-10T18:58:07.062Z24/30 Flush
208882023-12-10T18:58:07.065Z25/30 Write block 216 len 36864 data: 17 17 15 14 15 18 18 16 16
208892023-12-10T18:58:07.065Z26/30 Read block 515 len 8192
208902023-12-10T18:58:07.068Z27/30 Write block 157 len 16384 data: 13 11 13 11
208912023-12-10T18:58:07.068Z28/30 Write block 434 len 16384 data: 7 7 5 6
208922023-12-10T18:58:07.071Z29/30 Write block 332 len 36864 data: 15 14 12 11 12 12 10 13 12
208932023-12-10T18:58:07.071Z30/30 Read block 0 len 16384
208942023-12-10T18:58:07.074Z----------------------------------------------------------------
208952023-12-10T18:58:07.074Z Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:24
208962023-12-10T18:58:07.074ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
208972023-12-10T18:58:07.074Z 7 Acked 1006 Write 10 Done Done Done false
208982023-12-10T18:58:07.075Z 8 Acked 1007 Read 10 Sent Done Done false
208992023-12-10T18:58:07.075Z 9 Acked 1008 Read 5 Sent Done Done false
209002023-12-10T18:58:07.075Z 10 Acked 1009 Read 7 Sent Done Done false
209012023-12-10T18:58:07.075Z 11 Acked 1010 Write 6 Sent Done Done false
209022023-12-10T18:58:07.075Z 12 Acked 1011 Write 9 Sent Done Done false
209032023-12-10T18:58:07.075Z 13 Acked 1012 Write 7 Sent Done Done false
209042023-12-10T18:58:07.075Z 14 Acked 1013 Read 3 Sent Done Done false
209052023-12-10T18:58:07.075Z 15 Acked 1014 Write 5 Sent Done Done false
209062023-12-10T18:58:07.075Z 16 Acked 1015 Flush 0 Sent Done Done false
209072023-12-10T18:58:07.075Z 17 Acked 1016 Write 10 Sent Done Done false
209082023-12-10T18:58:07.075Z 18 Acked 1017 Read 9 Sent Done Done false
209092023-12-10T18:58:07.076Z 19 Acked 1018 Read 1 Sent Done Done false
209102023-12-10T18:58:07.076Z 20 Acked 1019 Read 5 Sent Done Done false
209112023-12-10T18:58:07.076Z 21 Acked 1020 Read 6 Sent Done Done false
209122023-12-10T18:58:07.076Z 22 Acked 1021 Write 6 Sent Done Done false
209132023-12-10T18:58:07.076Z 23 Acked 1022 Read 8 Sent Done Done false
209142023-12-10T18:58:07.076Z 24 Acked 1023 Flush 0 Sent Done Done false
209152023-12-10T18:58:07.076Z 25 Acked 1024 Write 9 Sent Done Done false
209162023-12-10T18:58:07.076Z 26 Acked 1025 Read 2 Sent Done Done false
209172023-12-10T18:58:07.078Z 27 Acked 1026 Write 4 Sent Done Done false
209182023-12-10T18:58:07.079Z 28 Acked 1027 Write 4 Sent Done Done false
209192023-12-10T18:58:07.079Z 29 Acked 1028 Write 9 Sent Done Done false
209202023-12-10T18:58:07.079Z 30 Acked 1029 Read 4 Sent Done Done false
209212023-12-10T18:58:07.079Z STATES DS:0 DS:1 DS:2 TOTAL
209222023-12-10T18:58:07.079Z New 0 0 0 0
209232023-12-10T18:58:07.079Z Sent 23 0 0 23
209242023-12-10T18:58:07.079Z Done 1 24 24 49
209252023-12-10T18:58:07.079Z Skipped 0 0 0 0
209262023-12-10T18:58:07.079Z Error 0 0 0 0
209272023-12-10T18:58:07.079ZLast Flush: 1005 1023 1023
209282023-12-10T18:58:07.079ZDownstairs last five completed: 1005 1004 1003 1002 1001 1000
209292023-12-10T18:58:07.079ZUpstairs last five completed: 30 29 28 27 26 25
209302023-12-10T18:58:07.079ZWrote out file "/tmp/test_repair_verify.data"
209312023-12-10T18:58:07.082Z
209322023-12-10T18:58:07.088ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
209332023-12-10T18:58:07.112ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
209342023-12-10T18:58:07.112Z 0 000-019 166 166 166 290 290 290 F F F
209352023-12-10T18:58:07.112Z 1 020-039 166 166 166 290 290 290 F F F
209362023-12-10T18:58:07.113Z 2 040-059 169 169 169 296 296 296 F F F
209372023-12-10T18:58:07.113Z 3 060-079 169 170 170 296 298 298 F F F <---
209382023-12-10T18:58:07.113Z 4 080-099 163 163 163 285 285 285 F F F
209392023-12-10T18:58:07.113Z 5 100-119 162 162 162 283 283 283 F F F
209402023-12-10T18:58:07.116Z 6 120-139 164 164 164 287 287 287 F F F
209412023-12-10T18:58:07.116Z 7 140-159 163 163 163 285 285 285 F T T <---
209422023-12-10T18:58:07.116Z 8 160-179 166 166 166 292 292 292 F T T <---
209432023-12-10T18:58:07.116Z 9 180-199 168 168 168 294 294 294 F F F
209442023-12-10T18:58:07.116Z 10 200-219 168 168 168 294 294 294 F T T <---
209452023-12-10T18:58:07.116Z 11 220-239 170 170 170 297 297 297 F T T <---
209462023-12-10T18:58:07.116Z 12 240-259 170 170 170 297 297 297 F F F
209472023-12-10T18:58:07.116Z 13 260-279 169 169 169 296 296 296 F F F
209482023-12-10T18:58:07.116Z 14 280-299 168 170 170 295 299 299 F F F <---
209492023-12-10T18:58:07.116Z 15 300-319 168 170 170 295 299 299 T F F <---
209502023-12-10T18:58:07.117Z 16 320-339 164 170 170 287 298 298 T T T <---
209512023-12-10T18:58:07.117Z 17 340-359 169 170 170 296 298 298 F T T <---
209522023-12-10T18:58:07.117Z 18 360-379 167 170 170 293 298 298 F F F <---
209532023-12-10T18:58:07.117Z 19 380-399 169 169 169 296 296 296 F F F
209542023-12-10T18:58:07.117Z 20 400-419 170 170 170 297 297 297 F F F
209552023-12-10T18:58:07.117Z 21 420-439 161 170 170 282 298 298 F T T <---
209562023-12-10T18:58:07.117Z 22 440-459 162 162 162 283 283 283 F F F
209572023-12-10T18:58:07.118Z 23 460-479 162 170 170 284 299 299 F F F <---
209582023-12-10T18:58:07.118Z 24 480-499 166 170 170 290 299 299 F F F <---
209592023-12-10T18:58:07.118Z 25 500-519 164 164 164 286 286 286 F F F
209602023-12-10T18:58:07.118Z 26 520-539 166 170 170 290 298 298 F F F <---
209612023-12-10T18:58:07.118Z 27 540-559 170 170 170 297 297 297 F F F
209622023-12-10T18:58:07.118Z 28 560-579 170 170 170 297 297 297 F F F
209632023-12-10T18:58:07.118Z 29 580-599 164 164 164 286 286 286 F F F
209642023-12-10T18:58:07.118ZMax gen: 170, Max flush: 299
209652023-12-10T18:58:07.118ZError: Difference in extent metadata found!
209662023-12-10T18:58:07.121ZA Difference in extent metadata is expected here
209672023-12-10T18:58:07.121ZOn loop 85
209682023-12-10T18:58:07.121Z
209692023-12-10T18:58:07.121ZVerifying data now
209702023-12-10T18:58:12.461ZNo change after verify
209712023-12-10T18:58:12.461ZLoop: 85 Downstairs dump after verify (and repair):
209722023-12-10T18:58:12.484ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
209732023-12-10T18:58:12.484Z 0 000-019 166 166 166 290 290 290 F F F
209742023-12-10T18:58:12.484Z 1 020-039 166 166 166 290 290 290 F F F
209752023-12-10T18:58:12.484Z 2 040-059 169 169 169 296 296 296 F F F
209762023-12-10T18:58:12.487Z 3 060-079 170 170 170 298 298 298 F F F
209772023-12-10T18:58:12.487Z 4 080-099 163 163 163 285 285 285 F F F
209782023-12-10T18:58:12.523Z 5 100-119 162 162 162 283 283 283 F F F
209792023-12-10T18:58:12.523Z 6 120-139 164 164 164 287 287 287 F F F
209802023-12-10T18:58:12.523Z 7 140-159 171 171 171 300 300 300 F F F
209812023-12-10T18:58:12.523Z 8 160-179 171 171 171 300 300 300 F F F
209822023-12-10T18:58:12.523Z 9 180-199 168 168 168 294 294 294 F F F
209832023-12-10T18:58:12.523Z 10 200-219 171 171 171 300 300 300 F F F
209842023-12-10T18:58:12.523Z 11 220-239 171 171 171 300 300 300 F F F
209852023-12-10T18:58:12.523Z 12 240-259 170 170 170 297 297 297 F F F
209862023-12-10T18:58:12.523Z 13 260-279 169 169 169 296 296 296 F F F
209872023-12-10T18:58:12.523Z 14 280-299 170 170 170 299 299 299 F F F
209882023-12-10T18:58:12.523Z 15 300-319 170 170 170 299 299 299 F F F
209892023-12-10T18:58:12.523Z 16 320-339 171 171 171 300 300 300 F F F
209902023-12-10T18:58:12.523Z 17 340-359 171 171 171 300 300 300 F F F
209912023-12-10T18:58:12.523Z 18 360-379 170 170 170 298 298 298 F F F
209922023-12-10T18:58:12.523Z 19 380-399 169 169 169 296 296 296 F F F
209932023-12-10T18:58:12.523Z 20 400-419 170 170 170 297 297 297 F F F
209942023-12-10T18:58:12.524Z 21 420-439 171 171 171 300 300 300 F F F
209952023-12-10T18:58:12.524Z 22 440-459 162 162 162 283 283 283 F F F
209962023-12-10T18:58:12.524Z 23 460-479 170 170 170 299 299 299 F F F
209972023-12-10T18:58:12.524Z 24 480-499 170 170 170 299 299 299 F F F
209982023-12-10T18:58:12.524Z 25 500-519 164 164 164 286 286 286 F F F
209992023-12-10T18:58:12.524Z 26 520-539 170 170 170 298 298 298 F F F
210002023-12-10T18:58:12.524Z 27 540-559 170 170 170 297 297 297 F F F
210012023-12-10T18:58:12.524Z 28 560-579 170 170 170 297 297 297 F F F
210022023-12-10T18:58:12.524Z 29 580-599 164 164 164 286 286 286 F F F
210032023-12-10T18:58:12.524ZMax gen: 171, Max flush: 300
210042023-12-10T18:58:12.532Z
210052023-12-10T18:58:12.532ZBegin loop 86
210062023-12-10T18:58:12.532ZDownstairs to restart: 0
210072023-12-10T18:58:12.532Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.517526853Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850}
210082023-12-10T18:58:12.532Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\nCrucible runtime is spawned
210092023-12-10T18:58:12.532Z git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.518254623Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850}
210102023-12-10T18:58:12.533Z{"msg":"The guest has requested activation with gen:172
210112023-12-10T18:58:12.533ZUpstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.518507601Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850}
210122023-12-10T18:58:12.534Z{"msg":"Crucible 698073db-499d-4809-a35d-f0fb183aa681 has session id: 759c570a-8424-497c-941c-b43964f48b15","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.518596608Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210132023-12-10T18:58:12.534Z{"msg":"Upstairs opts: Upstairs UUID: 698073db-499d-4809-a35d-f0fb183aa681, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.518700234Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210142023-12-10T18:58:12.534Z{"msg":"Crucible stats registered with UUID: 698073db-499d-4809-a35d-f0fb183aa681","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.518744503Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210152023-12-10T18:58:12.534Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.518780161Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15","task":"up_listen"}
210162023-12-10T18:58:12.535Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.51881856Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210172023-12-10T18:58:12.535Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.518852845Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"looper":"1","session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210182023-12-10T18:58:12.535Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.518890965Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210192023-12-10T18:58:12.535Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.518926555Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"looper":"0","session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210202023-12-10T18:58:12.535Z{"msg":"[1] 698073db-499d-4809-a35d-f0fb183aa681 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.519007064Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"looper":"1","session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210212023-12-10T18:58:12.535Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.519047824Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"looper":"2","session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210222023-12-10T18:58:12.535Z{"msg":"[0] 698073db-499d-4809-a35d-f0fb183aa681 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.519137059Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"looper":"0","session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210232023-12-10T18:58:12.535Z{"msg":"[2] 698073db-499d-4809-a35d-f0fb183aa681 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.519177751Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"looper":"2","session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210242023-12-10T18:58:12.535Z{"msg":"698073db-499d-4809-a35d-f0fb183aa681 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.519430146Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210252023-12-10T18:58:12.535Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.519492931Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210262023-12-10T18:58:12.535Z{"msg":"Set desired generation to :172","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.519525809Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210272023-12-10T18:58:12.536Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.519870828Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210282023-12-10T18:58:12.536Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.519942067Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210292023-12-10T18:58:12.536Z{"msg":"[1] 698073db-499d-4809-a35d-f0fb183aa681 (759c570a-8424-497c-941c-b43964f48b15) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.520194411Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210302023-12-10T18:58:12.536Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.520373962Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210312023-12-10T18:58:12.536Z{"msg":"[1] client is_active_req TRUE, promote! session 759c570a-8424-497c-941c-b43964f48b15","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.520470784Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210322023-12-10T18:58:12.536Z{"msg":"[2] 698073db-499d-4809-a35d-f0fb183aa681 (759c570a-8424-497c-941c-b43964f48b15) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.520519277Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210332023-12-10T18:58:12.536Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.520610178Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210342023-12-10T18:58:12.536Z{"msg":"[2] client is_active_req TRUE, promote! session 759c570a-8424-497c-941c-b43964f48b15","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.520655321Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210352023-12-10T18:58:12.536Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.521088496Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210362023-12-10T18:58:12.537Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.521193574Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210372023-12-10T18:58:12.538Z{"msg":"698073db-499d-4809-a35d-f0fb183aa681 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.521296338Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210382023-12-10T18:58:12.538Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.521371976Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210392023-12-10T18:58:12.538Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.521450939Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210402023-12-10T18:58:12.538Z{"msg":"698073db-499d-4809-a35d-f0fb183aa681 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.521491959Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210412023-12-10T18:58:12.538Z{"msg":"[0] 698073db-499d-4809-a35d-f0fb183aa681 (759c570a-8424-497c-941c-b43964f48b15) New WaitActive WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.52156454Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210422023-12-10T18:58:12.538Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.521633893Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210432023-12-10T18:58:12.538Z{"msg":"[0] client is_active_req TRUE, promote! session 759c570a-8424-497c-941c-b43964f48b15","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.521704489Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210442023-12-10T18:58:12.538Z{"msg":"[2] 698073db-499d-4809-a35d-f0fb183aa681 (759c570a-8424-497c-941c-b43964f48b15) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.521991715Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210452023-12-10T18:58:12.539Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.522051689Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210462023-12-10T18:58:12.539Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:12.5220769Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210472023-12-10T18:58:12.539Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.522097584Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210482023-12-10T18:58:12.539Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.522244481Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210492023-12-10T18:58:12.539Z{"msg":"698073db-499d-4809-a35d-f0fb183aa681 WaitActive WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.522298042Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210502023-12-10T18:58:12.539Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.522322103Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210512023-12-10T18:58:12.539Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.522342134Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210522023-12-10T18:58:12.539Z{"msg":"[1] 698073db-499d-4809-a35d-f0fb183aa681 (759c570a-8424-497c-941c-b43964f48b15) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.522530795Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210532023-12-10T18:58:12.539Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.522596845Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210542023-12-10T18:58:12.539Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:12.522621163Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210552023-12-10T18:58:12.539Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.522641901Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210562023-12-10T18:58:12.539Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.522662522Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210572023-12-10T18:58:12.539Z{"msg":"698073db-499d-4809-a35d-f0fb183aa681 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.522683061Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210582023-12-10T18:58:12.539Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.522703863Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210592023-12-10T18:58:12.539Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.522723957Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210602023-12-10T18:58:12.539Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.522765774Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210612023-12-10T18:58:12.539Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.522793594Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210622023-12-10T18:58:12.540Z{"msg":"698073db-499d-4809-a35d-f0fb183aa681 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.522835106Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210632023-12-10T18:58:12.540Z{"msg":"[0] 698073db-499d-4809-a35d-f0fb183aa681 (759c570a-8424-497c-941c-b43964f48b15) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.523331614Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210642023-12-10T18:58:12.540Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.523401281Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210652023-12-10T18:58:12.540Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:12.523428252Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210662023-12-10T18:58:12.540Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.523448426Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210672023-12-10T18:58:12.540Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.523490862Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210682023-12-10T18:58:12.540Z{"msg":"698073db-499d-4809-a35d-f0fb183aa681 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.523514302Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210692023-12-10T18:58:12.541Z{"msg":"[0]R flush_numbers[0..12]: [290, 290, 296, 298, 285, 283, 287, 300, 300, 294, 300, 300]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.523556807Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210702023-12-10T18:58:12.541Z{"msg":"[0]R generation[0..12]: [166, 166, 169, 170, 163, 162, 164, 171, 171, 168, 171, 171]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.523580691Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210712023-12-10T18:58:12.541Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.523622979Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210722023-12-10T18:58:12.541Z{"msg":"[1]R flush_numbers[0..12]: [290, 290, 296, 298, 285, 283, 287, 300, 300, 294, 300, 300]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.523646739Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210732023-12-10T18:58:12.541Z{"msg":"[1]R generation[0..12]: [166, 166, 169, 170, 163, 162, 164, 171, 171, 168, 171, 171]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.52369617Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210742023-12-10T18:58:12.541Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:172
210752023-12-10T18:58:12.541Z,"time":"Wait for a query_work_queue command to finish before sending IO
210762023-12-10T18:58:12.541Z2023-12-10T18:58:12.523720499Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210772023-12-10T18:58:12.541Z{"msg":"[2]R flush_numbers[0..12]: [290, 290, 296, 298, 285, 283, 287, 300, 300, 294, 300, 300]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.523744886Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210782023-12-10T18:58:12.542Z{"msg":"[2]R generation[0..12]: [166, 166, 169, 170, 163, 162, 164, 171, 171, 168, 171, 171]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.523797014Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210792023-12-10T18:58:12.542Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.523821036Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210802023-12-10T18:58:12.542Z{"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.523840685Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210812023-12-10T18:58:12.542Z{"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.5238607Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210822023-12-10T18:58:12.542Z{"msg":"Next flush: 301","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.52392726Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210832023-12-10T18:58:12.542Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.523950056Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210842023-12-10T18:58:12.542Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.523985571Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210852023-12-10T18:58:12.542Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.52402567Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210862023-12-10T18:58:12.542Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.524082644Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210872023-12-10T18:58:12.542Z{"msg":"698073db-499d-4809-a35d-f0fb183aa681 is now active with session: 759c570a-8424-497c-941c-b43964f48b15","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.524116742Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210882023-12-10T18:58:12.542Z{"msg":"698073db-499d-4809-a35d-f0fb183aa681 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.524184578Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210892023-12-10T18:58:12.542Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.524211493Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210902023-12-10T18:58:12.543Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.524259992Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210912023-12-10T18:58:12.543Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.524282067Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210922023-12-10T18:58:12.543Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.524329602Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210932023-12-10T18:58:12.543Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.524351792Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210942023-12-10T18:58:12.543Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.524396522Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210952023-12-10T18:58:12.543Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.524418061Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210962023-12-10T18:58:12.543Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.524438022Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210972023-12-10T18:58:12.543Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.524477613Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210982023-12-10T18:58:12.543Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.524697667Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
210992023-12-10T18:58:12.543Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:12.52477371Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1850,"session_id":"759c570a-8424-497c-941c-b43964f48b15"}
211002023-12-10T18:58:12.543ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
211012023-12-10T18:58:12.543ZLoading write count information from file "/tmp/test_repair_verify.data"
211022023-12-10T18:58:12.543ZRun Repair workload
211032023-12-10T18:58:12.543Z01/30 Write block 242 len 32768 data: 21 18 19 19 19 17 18 18
211042023-12-10T18:58:12.547Z02/30 Read block 125 len 24576
211052023-12-10T18:58:12.547Z03/30 Write block 387 len 12288 data: 15 14 11
211062023-12-10T18:58:12.547Z04/30 Read block 487 len 40960
211072023-12-10T18:58:12.547Z05/30 Write block 370 len 20480 data: 9 12 13 14 14
211082023-12-10T18:58:12.547Z06/30 Write block 356 len 40960 data: 17 15 14 16 14 13 12 12 8 8
211092023-12-10T18:58:12.547Z07/30 Flush
211102023-12-10T18:58:12.547Z08/30 Write block 162 len 40960 data: 18 18 17 16 18 17 17 16 15 16
211112023-12-10T18:58:12.548Z09/30 Read block 250 len 36864
211122023-12-10T18:58:12.550Z10/30 Flush
211132023-12-10T18:58:12.554Z11/30 Write block 405 len 20480 data: 17 14 14 17 15
211142023-12-10T18:58:12.554Z12/30 Read block 94 len 16384
211152023-12-10T18:58:12.554Z13/30 Read block 406 len 28672
211162023-12-10T18:58:12.559Z14/30 Read block 91 len 20480
211172023-12-10T18:58:12.564Z15/30 Flush
211182023-12-10T18:58:12.564Z16/30 Write block 466 len 4096 data: 11
211192023-12-10T18:58:12.564Z17/30 Write block 411 len 12288 data: 11 10 9
211202023-12-10T18:58:12.564Z18/30 Write block 86 len 20480 data: 22 20 20 18 14
211212023-12-10T18:58:12.564Z19/30 Read block 39 len 28672
211222023-12-10T18:58:12.568Z20/30 Write block 183 len 24576 data: 12 9 8 6 6 7
211232023-12-10T18:58:12.568Z21/30 Read block 163 len 8192
211242023-12-10T18:58:12.568Z22/30 Read block 144 len 24576
211252023-12-10T18:58:12.571Z23/30 Read block 73 len 16384
211262023-12-10T18:58:12.574Z24/30 Read block 295 len 4096
211272023-12-10T18:58:12.575Z25/30 Read block 139 len 32768
211282023-12-10T18:58:12.577Z26/30 Read block 43 len 28672
211292023-12-10T18:58:12.580Z27/30 Write block 24 len 32768 data: 15 14 13 9 8 7 10 13
211302023-12-10T18:58:12.580Z28/30 Read block 117 len 4096
211312023-12-10T18:58:12.583Z29/30 Write block 516 len 8192 data: 10 10
211322023-12-10T18:58:12.584Z30/30 Read block 198 len 28672
211332023-12-10T18:58:12.587Z----------------------------------------------------------------
211342023-12-10T18:58:12.587Z Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30
211352023-12-10T18:58:12.587ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
211362023-12-10T18:58:12.587Z 1 Acked 1000 Write 8 Done Done Done false
211372023-12-10T18:58:12.587Z 2 Acked 1001 Read 6 Done Done Done false
211382023-12-10T18:58:12.587Z 3 Acked 1002 Write 3 Done Done Done false
211392023-12-10T18:58:12.587Z 4 Acked 1003 Read 10 Done Done Done false
211402023-12-10T18:58:12.587Z 5 Acked 1004 Write 5 Sent Done Done false
211412023-12-10T18:58:12.588Z 6 Acked 1005 Write 10 Sent Done Done false
211422023-12-10T18:58:12.588Z 7 Acked 1006 Flush 0 Sent Done Done false
211432023-12-10T18:58:12.588Z 8 Acked 1007 Write 10 Sent Done Done false
211442023-12-10T18:58:12.588Z 9 Acked 1008 Read 9 Sent Done Done false
211452023-12-10T18:58:12.588Z 10 Acked 1009 Flush 0 Sent Done Done false
211462023-12-10T18:58:12.588Z 11 Acked 1010 Write 5 Sent Done Done false
211472023-12-10T18:58:12.588Z 12 Acked 1011 Read 4 Sent Done Done false
211482023-12-10T18:58:12.588Z 13 Acked 1012 Read 7 Sent Done Done false
211492023-12-10T18:58:12.589Z 14 Acked 1013 Read 5 Sent Done Done false
211502023-12-10T18:58:12.589Z 15 Acked 1014 Flush 0 Sent Done Done false
211512023-12-10T18:58:12.589Z 16 Acked 1015 Write 1 Sent Done Done false
211522023-12-10T18:58:12.589Z 17 Acked 1016 Write 3 Sent Done Done false
211532023-12-10T18:58:12.589Z 18 Acked 1017 Write 5 Sent Done Done false
211542023-12-10T18:58:12.589Z 19 Acked 1018 Read 7 Sent Done Done false
211552023-12-10T18:58:12.589Z 20 Acked 1019 Write 6 Sent Done Done false
211562023-12-10T18:58:12.589Z 21 Acked 1020 Read 2 Sent Done Done false
211572023-12-10T18:58:12.589Z 22 Acked 1021 Read 6 Sent Done Done false
211582023-12-10T18:58:12.589Z 23 Acked 1022 Read 4 Sent Done Done false
211592023-12-10T18:58:12.589Z 24 Acked 1023 Read 1 Sent Done Done false
211602023-12-10T18:58:12.589Z 25 Acked 1024 Read 8 Sent Done Done false
211612023-12-10T18:58:12.589Z 26 Acked 1025 Read 7 Sent Done Done false
211622023-12-10T18:58:12.589Z 27 Acked 1026 Write 8 Sent Done Done false
211632023-12-10T18:58:12.589Z 28 Acked 1027 Read 1 Sent Done Done false
211642023-12-10T18:58:12.589Z 29 Acked 1028 Write 2 Sent Done Done false
211652023-12-10T18:58:12.590Z 30 Acked 1029 Read 7 Sent Done Done false
211662023-12-10T18:58:12.590Z STATES DS:0 DS:1 DS:2 TOTAL
211672023-12-10T18:58:12.590Z New 0 0 0 0
211682023-12-10T18:58:12.590Z Sent 26 0 0 26
211692023-12-10T18:58:12.590Z Done 4 30 30 64
211702023-12-10T18:58:12.590Z Skipped 0 0 0 0
211712023-12-10T18:58:12.590Z Error 0 0 0 0
211722023-12-10T18:58:12.590ZLast Flush: 0 1014 1014
211732023-12-10T18:58:12.590ZDownstairs last five completed:
211742023-12-10T18:58:12.590ZUpstairs last five completed: 30 29 28 27 26 25
211752023-12-10T18:58:12.590ZWrote out file "/tmp/test_repair_verify.data"
211762023-12-10T18:58:12.593Z
211772023-12-10T18:58:12.598ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
211782023-12-10T18:58:12.625ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
211792023-12-10T18:58:12.625Z 0 000-019 166 166 166 290 290 290 F F F
211802023-12-10T18:58:12.625Z 1 020-039 166 166 166 290 290 290 F T T <---
211812023-12-10T18:58:12.625Z 2 040-059 169 169 169 296 296 296 F F F
211822023-12-10T18:58:12.625Z 3 060-079 170 170 170 298 298 298 F F F
211832023-12-10T18:58:12.625Z 4 080-099 163 163 163 285 285 285 F T T <---
211842023-12-10T18:58:12.629Z 5 100-119 162 162 162 283 283 283 F F F
211852023-12-10T18:58:12.629Z 6 120-139 164 164 164 287 287 287 F F F
211862023-12-10T18:58:12.629Z 7 140-159 171 171 171 300 300 300 F F F
211872023-12-10T18:58:12.629Z 8 160-179 171 172 172 300 302 302 F F F <---
211882023-12-10T18:58:12.629Z 9 180-199 168 168 168 294 294 294 F T T <---
211892023-12-10T18:58:12.629Z 10 200-219 171 171 171 300 300 300 F F F
211902023-12-10T18:58:12.629Z 11 220-239 171 171 171 300 300 300 F F F
211912023-12-10T18:58:12.629Z 12 240-259 170 172 172 297 301 301 T F F <---
211922023-12-10T18:58:12.629Z 13 260-279 169 169 169 296 296 296 F F F
211932023-12-10T18:58:12.629Z 14 280-299 170 170 170 299 299 299 F F F
211942023-12-10T18:58:12.629Z 15 300-319 170 170 170 299 299 299 F F F
211952023-12-10T18:58:12.629Z 16 320-339 171 171 171 300 300 300 F F F
211962023-12-10T18:58:12.629Z 17 340-359 171 172 172 300 301 301 F F F <---
211972023-12-10T18:58:12.631Z 18 360-379 170 172 172 298 301 301 F F F <---
211982023-12-10T18:58:12.631Z 19 380-399 169 172 172 296 301 301 T F F <---
211992023-12-10T18:58:12.631Z 20 400-419 170 172 172 297 303 303 F T T <---
212002023-12-10T18:58:12.631Z 21 420-439 171 171 171 300 300 300 F F F
212012023-12-10T18:58:12.631Z 22 440-459 162 162 162 283 283 283 F F F
212022023-12-10T18:58:12.631Z 23 460-479 170 170 170 299 299 299 F T T <---
212032023-12-10T18:58:12.631Z 24 480-499 170 170 170 299 299 299 F F F
212042023-12-10T18:58:12.631Z 25 500-519 164 164 164 286 286 286 F T T <---
212052023-12-10T18:58:12.631Z 26 520-539 170 170 170 298 298 298 F F F
212062023-12-10T18:58:12.631Z 27 540-559 170 170 170 297 297 297 F F F
212072023-12-10T18:58:12.631Z 28 560-579 170 170 170 297 297 297 F F F
212082023-12-10T18:58:12.632Z 29 580-599 164 164 164 286 286 286 F F F
212092023-12-10T18:58:12.632ZMax gen: 172, Max flush: 303
212102023-12-10T18:58:12.632ZError: Difference in extent metadata found!
212112023-12-10T18:58:12.635ZA Difference in extent metadata is expected here
212122023-12-10T18:58:12.635ZOn loop 86
212132023-12-10T18:58:12.635Z
212142023-12-10T18:58:12.635ZVerifying data now
212152023-12-10T18:58:18.746ZNo change after verify
212162023-12-10T18:58:18.746ZLoop: 86 Downstairs dump after verify (and repair):
212172023-12-10T18:58:18.797ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
212182023-12-10T18:58:18.797Z 0 000-019 166 166 166 290 290 290 F F F
212192023-12-10T18:58:18.800Z 1 020-039 173 173 173 304 304 304 F F F
212202023-12-10T18:58:18.801Z 2 040-059 169 169 169 296 296 296 F F F
212212023-12-10T18:58:18.801Z 3 060-079 170 170 170 298 298 298 F F F
212222023-12-10T18:58:18.801Z 4 080-099 173 173 173 304 304 304 F F F
212232023-12-10T18:58:18.801Z 5 100-119 162 162 162 283 283 283 F F F
212242023-12-10T18:58:18.801Z 6 120-139 164 164 164 287 287 287 F F F
212252023-12-10T18:58:18.801Z 7 140-159 171 171 171 300 300 300 F F F
212262023-12-10T18:58:18.801Z 8 160-179 172 172 172 302 302 302 F F F
212272023-12-10T18:58:18.801Z 9 180-199 173 173 173 304 304 304 F F F
212282023-12-10T18:58:18.801Z 10 200-219 171 171 171 300 300 300 F F F
212292023-12-10T18:58:18.801Z 11 220-239 171 171 171 300 300 300 F F F
212302023-12-10T18:58:18.801Z 12 240-259 172 172 172 301 301 301 F F F
212312023-12-10T18:58:18.801Z 13 260-279 169 169 169 296 296 296 F F F
212322023-12-10T18:58:18.801Z 14 280-299 170 170 170 299 299 299 F F F
212332023-12-10T18:58:18.801Z 15 300-319 170 170 170 299 299 299 F F F
212342023-12-10T18:58:18.801Z 16 320-339 171 171 171 300 300 300 F F F
212352023-12-10T18:58:18.802Z 17 340-359 172 172 172 301 301 301 F F F
212362023-12-10T18:58:18.802Z 18 360-379 172 172 172 301 301 301 F F F
212372023-12-10T18:58:18.802Z 19 380-399 172 172 172 301 301 301 F F F
212382023-12-10T18:58:18.802Z 20 400-419 173 173 173 304 304 304 F F F
212392023-12-10T18:58:18.802Z 21 420-439 171 171 171 300 300 300 F F F
212402023-12-10T18:58:18.802Z 22 440-459 162 162 162 283 283 283 F F F
212412023-12-10T18:58:18.802Z 23 460-479 173 173 173 304 304 304 F F F
212422023-12-10T18:58:18.802Z 24 480-499 170 170 170 299 299 299 F F F
212432023-12-10T18:58:18.802Z 25 500-519 173 173 173 304 304 304 F F F
212442023-12-10T18:58:18.802Z 26 520-539 170 170 170 298 298 298 F F F
212452023-12-10T18:58:18.802Z 27 540-559 170 170 170 297 297 297 F F F
212462023-12-10T18:58:18.802Z 28 560-579 170 170 170 297 297 297 F F F
212472023-12-10T18:58:18.802Z 29 580-599 164 164 164 286 286 286 F F F
212482023-12-10T18:58:18.802ZMax gen: 173, Max flush: 304
212492023-12-10T18:58:18.802Z
212502023-12-10T18:58:18.802ZBegin loop 87
212512023-12-10T18:58:18.802ZDownstairs to restart: 1
212522023-12-10T18:58:18.808Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.802453409Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858}
212532023-12-10T18:58:18.808Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.802982078Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858}
212542023-12-10T18:58:18.808Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.803015633Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858}
212552023-12-10T18:58:18.808Z{"msg":"Crucible 5a72036e-94b4-4a2b-9d2c-2f9d240fc69a has session id: f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.803123551Z","hostname":"Crucible runtime is spawned
212562023-12-10T18:58:18.808Zip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212572023-12-10T18:58:18.808Z{"The guest has requested activation with gen:174
212582023-12-10T18:58:18.808Zmsg":"Upstairs opts: Upstairs UUID: 5a72036e-94b4-4a2b-9d2c-2f9d240fc69a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.803368077Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212592023-12-10T18:58:18.808Z{"msg":"Crucible stats registered with UUID: 5a72036e-94b4-4a2b-9d2c-2f9d240fc69a","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.803449185Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212602023-12-10T18:58:18.808Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.803490862Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64","task":"up_listen"}
212612023-12-10T18:58:18.808Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.803549077Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212622023-12-10T18:58:18.808Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.803576303Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212632023-12-10T18:58:18.809Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.803610669Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"looper":"1","session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212642023-12-10T18:58:18.809Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.803660305Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"looper":"0","session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212652023-12-10T18:58:18.809Z{"msg":"[1] 5a72036e-94b4-4a2b-9d2c-2f9d240fc69a looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.803701815Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"looper":"1","session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212662023-12-10T18:58:18.809Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.803745059Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"looper":"2","session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212672023-12-10T18:58:18.809Z{"msg":"[0] 5a72036e-94b4-4a2b-9d2c-2f9d240fc69a looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.803830111Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"looper":"0","session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212682023-12-10T18:58:18.809Z{"msg":"[2] 5a72036e-94b4-4a2b-9d2c-2f9d240fc69a looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.803955693Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"looper":"2","session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212692023-12-10T18:58:18.809Z{"msg":"5a72036e-94b4-4a2b-9d2c-2f9d240fc69a active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.804316719Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212702023-12-10T18:58:18.809Z{"msg":"Set desired generation to :174","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.804382654Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212712023-12-10T18:58:18.809Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.804443339Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212722023-12-10T18:58:18.810Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.804782942Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212732023-12-10T18:58:18.810Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.80483996Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212742023-12-10T18:58:18.818Z{"msg":"[2] 5a72036e-94b4-4a2b-9d2c-2f9d240fc69a (f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.805399509Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212752023-12-10T18:58:18.818Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.805462065Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212762023-12-10T18:58:18.818Z{"msg":"[0] 5a72036e-94b4-4a2b-9d2c-2f9d240fc69a (f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.805490179Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212772023-12-10T18:58:18.818Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.805510843Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212782023-12-10T18:58:18.818Z{"msg":"[2] client is_active_req TRUE, promote! session f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.805543947Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212792023-12-10T18:58:18.818Z{"msg":"[0] client is_active_req TRUE, promote! session f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.805617658Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212802023-12-10T18:58:18.818Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.806347027Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212812023-12-10T18:58:18.818Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.806428204Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212822023-12-10T18:58:18.819Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.806464117Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212832023-12-10T18:58:18.819Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.806499604Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212842023-12-10T18:58:18.819Z{"msg":"5a72036e-94b4-4a2b-9d2c-2f9d240fc69a WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.806617164Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212852023-12-10T18:58:18.819Z{"msg":"[1] 5a72036e-94b4-4a2b-9d2c-2f9d240fc69a (f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.80667952Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212862023-12-10T18:58:18.819Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.806742971Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212872023-12-10T18:58:18.820Z{"msg":"[1] client is_active_req TRUE, promote! session f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.806769924Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212882023-12-10T18:58:18.820Z{"msg":"5a72036e-94b4-4a2b-9d2c-2f9d240fc69a WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.806802714Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212892023-12-10T18:58:18.821Z{"msg":"[0] 5a72036e-94b4-4a2b-9d2c-2f9d240fc69a (f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.807153451Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212902023-12-10T18:58:18.821Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.807214247Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212912023-12-10T18:58:18.824Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:18.807239786Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212922023-12-10T18:58:18.824Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.807266047Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212932023-12-10T18:58:18.824Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.807313611Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212942023-12-10T18:58:18.824Z{"msg":"5a72036e-94b4-4a2b-9d2c-2f9d240fc69a WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.807339225Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212952023-12-10T18:58:18.824Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.807382777Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212962023-12-10T18:58:18.824Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.807404984Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212972023-12-10T18:58:18.824Z{"msg":"[2] 5a72036e-94b4-4a2b-9d2c-2f9d240fc69a (f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.807533008Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212982023-12-10T18:58:18.824Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.807586024Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
212992023-12-10T18:58:18.824Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:18.80760911Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213002023-12-10T18:58:18.824Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.80762937Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213012023-12-10T18:58:18.824Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.807684993Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213022023-12-10T18:58:18.824Z{"msg":"5a72036e-94b4-4a2b-9d2c-2f9d240fc69a WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.807806445Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213032023-12-10T18:58:18.824Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.807830331Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213042023-12-10T18:58:18.824Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.807871069Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213052023-12-10T18:58:18.824Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.807894891Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213062023-12-10T18:58:18.825Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.807940455Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213072023-12-10T18:58:18.825Z{"msg":"5a72036e-94b4-4a2b-9d2c-2f9d240fc69a WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.807963916Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213082023-12-10T18:58:18.825Z{"msg":"[1] 5a72036e-94b4-4a2b-9d2c-2f9d240fc69a (f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.808471124Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213092023-12-10T18:58:18.825Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.808556035Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213102023-12-10T18:58:18.825Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:18.808600883Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213112023-12-10T18:58:18.825Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.808687472Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213122023-12-10T18:58:18.825Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.808757321Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213132023-12-10T18:58:18.825Z{"msg":"5a72036e-94b4-4a2b-9d2c-2f9d240fc69a WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.808830604Z","The guest has finished waiting for activation with:174
213142023-12-10T18:58:18.825Zhostname":"ip-10-150-1-104.us-west-2.compute.internal"Wait for a query_work_queue command to finish before sending IO
213152023-12-10T18:58:18.825Z,"pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213162023-12-10T18:58:18.825Z{"msg":"[0]R flush_numbers[0..12]: [290, 304, 296, 298, 304, 283, 287, 300, 302, 304, 300, 300]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.808881262Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213172023-12-10T18:58:18.825Z{"msg":"[0]R generation[0..12]: [166, 173, 169, 170, 173, 162, 164, 171, 172, 173, 171, 171]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.808923331Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213182023-12-10T18:58:18.825Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.809003914Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213192023-12-10T18:58:18.825Z{"msg":"[1]R flush_numbers[0..12]: [290, 304, 296, 298, 304, 283, 287, 300, 302, 304, 300, 300]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.809048211Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213202023-12-10T18:58:18.825Z{"msg":"[1]R generation[0..12]: [166, 173, 169, 170, 173, 162, 164, 171, 172, 173, 171, 171]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.809137077Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213212023-12-10T18:58:18.825Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.80918253Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213222023-12-10T18:58:18.825Z{"msg":"[2]R flush_numbers[0..12]: [290, 304, 296, 298, 304, 283, 287, 300, 302, 304, 300, 300]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.809225358Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213232023-12-10T18:58:18.826Z{"msg":"[2]R generation[0..12]: [166, 173, 169, 170, 173, 162, 164, 171, 172, 173, 171, 171]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.809253901Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213242023-12-10T18:58:18.826Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.809275915Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213252023-12-10T18:58:18.826Z{"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.809296197Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213262023-12-10T18:58:18.826Z{"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.809347138Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213272023-12-10T18:58:18.826Z{"msg":"Next flush: 305","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.809419118Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213282023-12-10T18:58:18.826Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.809441593Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213292023-12-10T18:58:18.826Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.809465391Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213302023-12-10T18:58:18.826Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.809511361Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213312023-12-10T18:58:18.826Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.809535632Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213322023-12-10T18:58:18.826Z{"msg":"5a72036e-94b4-4a2b-9d2c-2f9d240fc69a is now active with session: f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.809592133Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213332023-12-10T18:58:18.826Z{"msg":"5a72036e-94b4-4a2b-9d2c-2f9d240fc69a Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.809615128Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213342023-12-10T18:58:18.826Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.809635264Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213352023-12-10T18:58:18.826Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.809671114Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213362023-12-10T18:58:18.826Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.80975376Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213372023-12-10T18:58:18.826Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.80978284Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213382023-12-10T18:58:18.826Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.809805273Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213392023-12-10T18:58:18.826Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.809825013Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213402023-12-10T18:58:18.827Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.809850196Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213412023-12-10T18:58:18.827Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.809878421Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213422023-12-10T18:58:18.827Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.809899902Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213432023-12-10T18:58:18.827Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.809919555Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213442023-12-10T18:58:18.827Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:18.809980443Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1858,"session_id":"f4bed9fe-1fa6-45c1-893c-3e8d8e8e0b64"}
213452023-12-10T18:58:18.827ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
213462023-12-10T18:58:18.827ZLoading write count information from file "/tmp/test_repair_verify.data"
213472023-12-10T18:58:18.827ZRun Repair workload
213482023-12-10T18:58:18.827Z01/30 Write block 511 len 36864 data: 9 8 9 9 8 11 11 10 11
213492023-12-10T18:58:18.831Z02/30 Read block 319 len 20480
213502023-12-10T18:58:18.831Z03/30 Read block 36 len 20480
213512023-12-10T18:58:18.831Z04/30 Read block 306 len 32768
213522023-12-10T18:58:18.831Z05/30 Read block 330 len 20480
213532023-12-10T18:58:18.831Z06/30 Read block 226 len 12288
213542023-12-10T18:58:18.831Z07/30 Read block 443 len 12288
213552023-12-10T18:58:18.831Z08/30 Write block 84 len 8192 data: 20 19
213562023-12-10T18:58:18.831Z09/30 Write block 402 len 4096 data: 17
213572023-12-10T18:58:18.831Z10/30 Flush
213582023-12-10T18:58:18.834Z11/30 Write block 19 len 4096 data: 15
213592023-12-10T18:58:18.834Z12/30 Flush
213602023-12-10T18:58:18.834Z13/30 Write block 541 len 24576 data: 21 21 20 18 14 12
213612023-12-10T18:58:18.837Z14/30 Write block 265 len 8192 data: 15 15
213622023-12-10T18:58:18.837Z15/30 Flush
213632023-12-10T18:58:18.840Z16/30 Write block 67 len 4096 data: 14
213642023-12-10T18:58:18.841Z17/30 Write block 566 len 32768 data: 16 15 12 13 15 13 11 12
213652023-12-10T18:58:18.841Z18/30 Write block 17 len 20480 data: 17 14 16 16 14
213662023-12-10T18:58:18.844Z19/30 Write block 115 len 20480 data: 15 16 15 15 14
213672023-12-10T18:58:18.844Z20/30 Write block 129 len 28672 data: 14 12 11 9 9 7 6
213682023-12-10T18:58:18.844Z21/30 Read block 309 len 36864
213692023-12-10T18:58:18.850Z22/30 Read block 133 len 20480
213702023-12-10T18:58:18.853Z23/30 Write block 409 len 40960 data: 16 14 12 11 10 10 10 11 12 13
213712023-12-10T18:58:18.853Z24/30 Read block 474 len 24576
213722023-12-10T18:58:18.853Z25/30 Read block 269 len 32768
213732023-12-10T18:58:18.856Z26/30 Write block 569 len 40960 data: 14 16 14 12 13 14 15 14 14 11
213742023-12-10T18:58:18.859Z27/30 Read block 280 len 32768
213752023-12-10T18:58:18.862Z28/30 Read block 219 len 40960
213762023-12-10T18:58:18.865Z29/30 Write block 352 len 8192 data: 19 16
213772023-12-10T18:58:18.865Z30/30 Write block 152 len 16384 data: 13 14 18 15
213782023-12-10T18:58:18.865Z----------------------------------------------------------------
213792023-12-10T18:58:18.866Z Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30
213802023-12-10T18:58:18.866ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
213812023-12-10T18:58:18.866Z 1 Acked 1000 Write 9 Done Done Done false
213822023-12-10T18:58:18.866Z 2 Acked 1001 Read 5 Done Done Done false
213832023-12-10T18:58:18.867Z 3 Acked 1002 Read 5 Done Done Done false
213842023-12-10T18:58:18.867Z 4 Acked 1003 Read 8 Done Done Done false
213852023-12-10T18:58:18.867Z 5 Acked 1004 Read 5 Done Done Done false
213862023-12-10T18:58:18.867Z 6 Acked 1005 Read 3 Done Sent Done false
213872023-12-10T18:58:18.867Z 7 Acked 1006 Read 3 Done Sent Done false
213882023-12-10T18:58:18.867Z 8 Acked 1007 Write 2 Done Sent Done false
213892023-12-10T18:58:18.867Z 9 Acked 1008 Write 1 Done Sent Done false
213902023-12-10T18:58:18.867Z 10 Acked 1009 Flush 0 Done Sent Done false
213912023-12-10T18:58:18.867Z 11 Acked 1010 Write 1 Done Sent Done false
213922023-12-10T18:58:18.867Z 12 Acked 1011 Flush 0 Done Sent Done false
213932023-12-10T18:58:18.867Z 13 Acked 1012 Write 6 Done Sent Done false
213942023-12-10T18:58:18.867Z 14 Acked 1013 Write 2 Done Sent Done false
213952023-12-10T18:58:18.867Z 15 Acked 1014 Flush 0 Done Sent Done false
213962023-12-10T18:58:18.867Z 16 Acked 1015 Write 1 Done Sent Done false
213972023-12-10T18:58:18.867Z 17 Acked 1016 Write 8 Done Sent Done false
213982023-12-10T18:58:18.867Z 18 Acked 1017 Write 5 Done Sent Done false
213992023-12-10T18:58:18.867Z 19 Acked 1018 Write 5 Done Sent Done false
214002023-12-10T18:58:18.868Z 20 Acked 1019 Write 7 Done Sent Done false
214012023-12-10T18:58:18.868Z 21 Acked 1020 Read 9 Done Sent Done false
214022023-12-10T18:58:18.868Z 22 Acked 1021 Read 5 Done Sent Done false
214032023-12-10T18:58:18.868Z 23 Acked 1022 Write 10 Done Sent Done false
214042023-12-10T18:58:18.868Z 24 Acked 1023 Read 6 Done Sent Done false
214052023-12-10T18:58:18.868Z 25 Acked 1024 Read 8 Done Sent Done false
214062023-12-10T18:58:18.868Z 26 Acked 1025 Write 10 Done Sent Done false
214072023-12-10T18:58:18.868Z 27 Acked 1026 Read 8 Done Sent Done false
214082023-12-10T18:58:18.868Z 28 Acked 1027 Read 10 Done Sent Done false
214092023-12-10T18:58:18.868Z 29 Acked 1028 Write 2 Done Sent Done false
214102023-12-10T18:58:18.868Z 30 Acked 1029 Write 4 Sent Sent Sent false
214112023-12-10T18:58:18.868Z STATES DS:0 DS:1 DS:2 TOTAL
214122023-12-10T18:58:18.868Z New 0 0 0 0
214132023-12-10T18:58:18.868Z Sent 1 25 1 27
214142023-12-10T18:58:18.868Z Done 29 5 29 63
214152023-12-10T18:58:18.868Z Skipped 0 0 0 0
214162023-12-10T18:58:18.868Z Error 0 0 0 0
214172023-12-10T18:58:18.871ZLast Flush: 1014 0 1014
214182023-12-10T18:58:18.872ZDownstairs last five completed:
214192023-12-10T18:58:18.872ZUpstairs last five completed: 30 29 28 27 26 25
214202023-12-10T18:58:18.872ZWrote out file "/tmp/test_repair_verify.data"
214212023-12-10T18:58:18.872Z
214222023-12-10T18:58:18.876ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
214232023-12-10T18:58:18.903ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
214242023-12-10T18:58:18.904Z 0 000-019 174 166 174 306 290 306 T F T <---
214252023-12-10T18:58:18.904Z 1 020-039 173 173 173 304 304 304 T F T <---
214262023-12-10T18:58:18.904Z 2 040-059 169 169 169 296 296 296 F F F
214272023-12-10T18:58:18.904Z 3 060-079 170 170 170 298 298 298 T F T <---
214282023-12-10T18:58:18.904Z 4 080-099 174 173 174 305 304 305 F F F <---
214292023-12-10T18:58:18.904Z 5 100-119 162 162 162 283 283 283 T F T <---
214302023-12-10T18:58:18.904Z 6 120-139 164 164 164 287 287 287 T F T <---
214312023-12-10T18:58:18.904Z 7 140-159 171 171 171 300 300 300 T F T <---
214322023-12-10T18:58:18.904Z 8 160-179 172 172 172 302 302 302 F F F
214332023-12-10T18:58:18.904Z 9 180-199 173 173 173 304 304 304 F F F
214342023-12-10T18:58:18.907Z 10 200-219 171 171 171 300 300 300 F F F
214352023-12-10T18:58:18.907Z 11 220-239 171 171 171 300 300 300 F F F
214362023-12-10T18:58:18.907Z 12 240-259 172 172 172 301 301 301 F F F
214372023-12-10T18:58:18.907Z 13 260-279 174 169 174 307 296 307 F F F <---
214382023-12-10T18:58:18.908Z 14 280-299 170 170 170 299 299 299 F F F
214392023-12-10T18:58:18.908Z 15 300-319 170 170 170 299 299 299 F F F
214402023-12-10T18:58:18.908Z 16 320-339 171 171 171 300 300 300 F F F
214412023-12-10T18:58:18.908Z 17 340-359 172 172 172 301 301 301 T F T <---
214422023-12-10T18:58:18.908Z 18 360-379 172 172 172 301 301 301 F F F
214432023-12-10T18:58:18.908Z 19 380-399 172 172 172 301 301 301 F F F
214442023-12-10T18:58:18.908Z 20 400-419 174 173 174 305 304 305 T F T <---
214452023-12-10T18:58:18.908Z 21 420-439 171 171 171 300 300 300 F F F
214462023-12-10T18:58:18.908Z 22 440-459 162 162 162 283 283 283 F F F
214472023-12-10T18:58:18.909Z 23 460-479 173 173 173 304 304 304 F F F
214482023-12-10T18:58:18.909Z 24 480-499 170 170 170 299 299 299 F F F
214492023-12-10T18:58:18.909Z 25 500-519 174 173 174 305 304 305 F T F <---
214502023-12-10T18:58:18.909Z 26 520-539 170 170 170 298 298 298 F F F
214512023-12-10T18:58:18.909Z 27 540-559 174 170 174 307 297 307 F F F <---
214522023-12-10T18:58:18.909Z 28 560-579 170 170 170 297 297 297 T F T <---
214532023-12-10T18:58:18.909Z 29 580-599 164 164 164 286 286 286 F F F
214542023-12-10T18:58:18.909ZMax gen: 174, Max flush: 307
214552023-12-10T18:58:18.909ZError: Difference in extent metadata found!
214562023-12-10T18:58:18.912ZA Difference in extent metadata is expected here
214572023-12-10T18:58:18.912ZOn loop 87
214582023-12-10T18:58:18.912Z
214592023-12-10T18:58:18.912ZVerifying data now
214602023-12-10T18:58:24.191ZNo change after verify
214612023-12-10T18:58:24.191ZLoop: 87 Downstairs dump after verify (and repair):
214622023-12-10T18:58:24.214ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
214632023-12-10T18:58:24.214Z 0 000-019 175 175 175 308 308 308 F F F
214642023-12-10T18:58:24.217Z 1 020-039 175 175 175 308 308 308 F F F
214652023-12-10T18:58:24.218Z 2 040-059 169 169 169 296 296 296 F F F
214662023-12-10T18:58:24.219Z 3 060-079 175 175 175 308 308 308 F F F
214672023-12-10T18:58:24.219Z 4 080-099 174 174 174 305 305 305 F F F
214682023-12-10T18:58:24.219Z 5 100-119 175 175 175 308 308 308 F F F
214692023-12-10T18:58:24.219Z 6 120-139 175 175 175 308 308 308 F F F
214702023-12-10T18:58:24.219Z 7 140-159 175 175 175 308 308 308 F F F
214712023-12-10T18:58:24.219Z 8 160-179 172 172 172 302 302 302 F F F
214722023-12-10T18:58:24.220Z 9 180-199 173 173 173 304 304 304 F F F
214732023-12-10T18:58:24.220Z 10 200-219 171 171 171 300 300 300 F F F
214742023-12-10T18:58:24.220Z 11 220-239 171 171 171 300 300 300 F F F
214752023-12-10T18:58:24.220Z 12 240-259 172 172 172 301 301 301 F F F
214762023-12-10T18:58:24.221Z 13 260-279 174 174 174 307 307 307 F F F
214772023-12-10T18:58:24.221Z 14 280-299 170 170 170 299 299 299 F F F
214782023-12-10T18:58:24.221Z 15 300-319 170 170 170 299 299 299 F F F
214792023-12-10T18:58:24.221Z 16 320-339 171 171 171 300 300 300 F F F
214802023-12-10T18:58:24.221Z 17 340-359 175 175 175 308 308 308 F F F
214812023-12-10T18:58:24.221Z 18 360-379 172 172 172 301 301 301 F F F
214822023-12-10T18:58:24.221Z 19 380-399 172 172 172 301 301 301 F F F
214832023-12-10T18:58:24.221Z 20 400-419 175 175 175 308 308 308 F F F
214842023-12-10T18:58:24.221Z 21 420-439 171 171 171 300 300 300 F F F
214852023-12-10T18:58:24.221Z 22 440-459 162 162 162 283 283 283 F F F
214862023-12-10T18:58:24.221Z 23 460-479 173 173 173 304 304 304 F F F
214872023-12-10T18:58:24.221Z 24 480-499 170 170 170 299 299 299 F F F
214882023-12-10T18:58:24.221Z 25 500-519 174 174 174 305 305 305 F F F
214892023-12-10T18:58:24.221Z 26 520-539 170 170 170 298 298 298 F F F
214902023-12-10T18:58:24.222Z 27 540-559 174 174 174 307 307 307 F F F
214912023-12-10T18:58:24.222Z 28 560-579 175 175 175 308 308 308 F F F
214922023-12-10T18:58:24.222Z 29 580-599 164 164 164 286 286 286 F F F
214932023-12-10T18:58:24.222ZMax gen: 175, Max flush: 308
214942023-12-10T18:58:24.226Z
214952023-12-10T18:58:24.226ZBegin loop 88
214962023-12-10T18:58:24.226ZDownstairs to restart: 1
214972023-12-10T18:58:24.249Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.246954554Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866}
214982023-12-10T18:58:24.249Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.247399632Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866}
214992023-12-10T18:58:24.253Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.247463278Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866}
215002023-12-10T18:58:24.254Z{"msg":"Crucible feb2b94e-8ba2-4034-b429-069cdd0a5425 has session id: 4d083d8e-0f7b-44e5-81ae-fe37803a325c","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.247524672Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215012023-12-10T18:58:24.254Z{"msg":"Upstairs opts: Upstairs UUID: feb2b94e-8ba2-4034-b429-069cdd0a5425, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30Crucible runtime is spawned
215022023-12-10T18:58:24.254Z,"time":"2023-12-10T18:58:24.247587474Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215032023-12-10T18:58:24.254Z{"msg":"Crucible stats registered with UUID: feb2b94e-8ba2-4034-b429-069cdd0a5425"The guest has requested activation with gen:176
215042023-12-10T18:58:24.254Z,"v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.247845793Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215052023-12-10T18:58:24.255Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.247917392Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c","task":"up_listen"}
215062023-12-10T18:58:24.255Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.247945068Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215072023-12-10T18:58:24.255Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.248003213Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"looper":"0","session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215082023-12-10T18:58:24.255Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.248054541Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215092023-12-10T18:58:24.255Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.248165234Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"looper":"2","session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215102023-12-10T18:58:24.255Z{"msg":"[0] feb2b94e-8ba2-4034-b429-069cdd0a5425 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.248210375Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"looper":"0","session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215112023-12-10T18:58:24.255Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.24840046Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"looper":"1","session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215122023-12-10T18:58:24.255Z{"msg":"[2] feb2b94e-8ba2-4034-b429-069cdd0a5425 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.248434869Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"looper":"2","session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215132023-12-10T18:58:24.255Z{"msg":"[1] feb2b94e-8ba2-4034-b429-069cdd0a5425 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.248492775Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"looper":"1","session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215142023-12-10T18:58:24.256Z{"msg":"feb2b94e-8ba2-4034-b429-069cdd0a5425 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.248910957Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215152023-12-10T18:58:24.256Z{"msg":"Set desired generation to :176","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.248997464Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215162023-12-10T18:58:24.257Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.249026705Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215172023-12-10T18:58:24.257Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.249286456Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215182023-12-10T18:58:24.257Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.249514706Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215192023-12-10T18:58:24.257Z{"msg":"[0] feb2b94e-8ba2-4034-b429-069cdd0a5425 (4d083d8e-0f7b-44e5-81ae-fe37803a325c) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.249734655Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215202023-12-10T18:58:24.257Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.249869721Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215212023-12-10T18:58:24.257Z{"msg":"[0] client is_active_req TRUE, promote! session 4d083d8e-0f7b-44e5-81ae-fe37803a325c","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.249968759Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215222023-12-10T18:58:24.257Z{"msg":"[2] feb2b94e-8ba2-4034-b429-069cdd0a5425 (4d083d8e-0f7b-44e5-81ae-fe37803a325c) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.25004163Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215232023-12-10T18:58:24.257Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.250118538Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215242023-12-10T18:58:24.257Z{"msg":"[2] client is_active_req TRUE, promote! session 4d083d8e-0f7b-44e5-81ae-fe37803a325c","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.250164168Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215252023-12-10T18:58:24.257Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.25049393Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215262023-12-10T18:58:24.257Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.250558277Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215272023-12-10T18:58:24.257Z{"msg":"feb2b94e-8ba2-4034-b429-069cdd0a5425 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.250607002Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215282023-12-10T18:58:24.257Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.250765219Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215292023-12-10T18:58:24.257Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.250842917Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215302023-12-10T18:58:24.257Z{"msg":"feb2b94e-8ba2-4034-b429-069cdd0a5425 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.250869571Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215312023-12-10T18:58:24.266Z{"msg":"[1] feb2b94e-8ba2-4034-b429-069cdd0a5425 (4d083d8e-0f7b-44e5-81ae-fe37803a325c) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.250905985Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215322023-12-10T18:58:24.266Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.25095421Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215332023-12-10T18:58:24.266Z{"msg":"[1] client is_active_req TRUE, promote! session 4d083d8e-0f7b-44e5-81ae-fe37803a325c","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.250997345Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215342023-12-10T18:58:24.266Z{"msg":"[0] feb2b94e-8ba2-4034-b429-069cdd0a5425 (4d083d8e-0f7b-44e5-81ae-fe37803a325c) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.25131155Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215352023-12-10T18:58:24.267Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.251419083Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215362023-12-10T18:58:24.267Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:24.251462372Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215372023-12-10T18:58:24.267Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.251502067Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215382023-12-10T18:58:24.267Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.251541463Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215392023-12-10T18:58:24.267Z{"msg":"feb2b94e-8ba2-4034-b429-069cdd0a5425 WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.251603281Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215402023-12-10T18:58:24.267Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.251643836Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215412023-12-10T18:58:24.267Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.251701751Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215422023-12-10T18:58:24.267Z{"msg":"[2] feb2b94e-8ba2-4034-b429-069cdd0a5425 (4d083d8e-0f7b-44e5-81ae-fe37803a325c) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.251796386Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215432023-12-10T18:58:24.267Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.251900655Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215442023-12-10T18:58:24.267Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:24.251961601Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215452023-12-10T18:58:24.268Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.251986575Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215462023-12-10T18:58:24.268Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.252011434Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215472023-12-10T18:58:24.268Z{"msg":"feb2b94e-8ba2-4034-b429-069cdd0a5425 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.252066398Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215482023-12-10T18:58:24.268Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.252091028Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215492023-12-10T18:58:24.268Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.25211544Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215502023-12-10T18:58:24.268Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.252170927Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215512023-12-10T18:58:24.268Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.25220072Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215522023-12-10T18:58:24.269Z{"msg":"feb2b94e-8ba2-4034-b429-069cdd0a5425 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.252258402Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215532023-12-10T18:58:24.269Z{"msg":"[1] feb2b94e-8ba2-4034-b429-069cdd0a5425 (4d083d8e-0f7b-44e5-81ae-fe37803a325c) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.252742849Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215542023-12-10T18:58:24.269Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.252820307Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215552023-12-10T18:58:24.272Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:24.252845704Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215562023-12-10T18:58:24.272Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.252867411Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215572023-12-10T18:58:24.272Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.252888232Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215582023-12-10T18:58:24.272Z{"msg":"feb2b94e-8ba2-4034-b429-069cdd0a5425 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.252909026Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215592023-12-10T18:58:24.272Z{"msg":"[0]R flush_numbers[0..12]: [308, 308, 296, 308, 305, 308, 308, 308, 302, 304, 300, 300]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.252952Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215602023-12-10T18:58:24.272Z{"msg":"[0]R generation[0..12]: [175, 175, 169, 175, 174, 175, 175, 175, 172, 173, 171, 171]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.25297687Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215612023-12-10T18:58:24.272Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.253020459Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215622023-12-10T18:58:24.272Z{"msg":"[1]R flush_numbers[0..12]: [308, 308, 296, 308, 305, 308, 308, 308, 302, 304, 300, 300]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.253044769Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215632023-12-10T18:58:24.272Z{"msg":"[1]R generation[0..12]: [175, 175, 169, 175, 174, 175, 175, 175, 172, 173, 171, 171]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.253087013Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215642023-12-10T18:58:24.272Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.2531114Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215652023-12-10T18:58:24.272Z{"msg":"[2]R flush_numbers[0..12]: [308, 308, 296, 308, 305, 308, 308, 308, 302, 304, 300, 300]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.253153408Z"The guest has finished waiting for activation with:176
215662023-12-10T18:58:24.272Z,"hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"Wait for a query_work_queue command to finish before sending IO
215672023-12-10T18:58:24.273Zsession_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215682023-12-10T18:58:24.273Z{"msg":"[2]R generation[0..12]: [175, 175, 169, 175, 174, 175, 175, 175, 172, 173, 171, 171]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.253183204Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215692023-12-10T18:58:24.273Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.253236008Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215702023-12-10T18:58:24.274Z{"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.253257878Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215712023-12-10T18:58:24.274Z{"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.253278111Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215722023-12-10T18:58:24.274Z{"msg":"Next flush: 309","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.253330347Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215732023-12-10T18:58:24.274Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.253352059Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215742023-12-10T18:58:24.274Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.253372363Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215752023-12-10T18:58:24.274Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.253393494Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215762023-12-10T18:58:24.274Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.253457468Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215772023-12-10T18:58:24.274Z{"msg":"feb2b94e-8ba2-4034-b429-069cdd0a5425 is now active with session: 4d083d8e-0f7b-44e5-81ae-fe37803a325c","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.253503857Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215782023-12-10T18:58:24.274Z{"msg":"feb2b94e-8ba2-4034-b429-069cdd0a5425 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.253577292Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215792023-12-10T18:58:24.274Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.253622859Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215802023-12-10T18:58:24.274Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.253701033Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215812023-12-10T18:58:24.274Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.253739719Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215822023-12-10T18:58:24.274Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.253785611Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215832023-12-10T18:58:24.274Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.253808433Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215842023-12-10T18:58:24.274Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.253829253Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215852023-12-10T18:58:24.274Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.253898601Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215862023-12-10T18:58:24.274Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.253924382Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215872023-12-10T18:58:24.275Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.253979758Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
215882023-12-10T18:58:24.275Z,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215892023-12-10T18:58:24.275Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.254100769Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215902023-12-10T18:58:24.275Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:24.254128454Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1866,"session_id":"4d083d8e-0f7b-44e5-81ae-fe37803a325c"}
215912023-12-10T18:58:24.275ZLoading write count information from file "/tmp/test_repair_verify.data"
215922023-12-10T18:58:24.275ZRun Repair workload
215932023-12-10T18:58:24.275Z01/30 Write block 366 len 32768 data: 9 9 9 10 10 13 14 15
215942023-12-10T18:58:24.275Z02/30 Read block 295 len 20480
215952023-12-10T18:58:24.275Z03/30 Write block 109 len 24576 data: 11 12 11 12 15 13
215962023-12-10T18:58:24.279Z04/30 Write block 258 len 20480 data: 9 5 9 11 15
215972023-12-10T18:58:24.279Z05/30 Read block 170 len 8192
215982023-12-10T18:58:24.279Z06/30 Write block 371 len 36864 data: 14 15 16 15 18 20 19 22 19
215992023-12-10T18:58:24.279Z07/30 Write block 523 len 28672 data: 14 14 14 16 15 15 16
216002023-12-10T18:58:24.279Z08/30 Write block 91 len 32768 data: 11 12 14 14 16 11 8 8
216012023-12-10T18:58:24.279Z09/30 Read block 452 len 20480
216022023-12-10T18:58:24.279Z10/30 Read block 158 len 20480
216032023-12-10T18:58:24.279Z11/30 Read block 81 len 32768
216042023-12-10T18:58:24.279Z12/30 Write block 563 len 4096 data: 15
216052023-12-10T18:58:24.279Z13/30 Write block 438 len 4096 data: 8
216062023-12-10T18:58:24.279Z14/30 Write block 180 len 28672 data: 8 10 13 13 10 9 7
216072023-12-10T18:58:24.279Z15/30 Write block 473 len 36864 data: 10 9 8 10 8 6 7 8 8
216082023-12-10T18:58:24.283Z16/30 Write block 413 len 28672 data: 11 11 11 12 13 14 13
216092023-12-10T18:58:24.283Z17/30 Write block 399 len 24576 data: 14 15 16 18 17 14
216102023-12-10T18:58:24.283Z18/30 Write block 467 len 32768 data: 13 12 15 13 12 11 11 10
216112023-12-10T18:58:24.286Z19/30 Write block 446 len 36864 data: 12 12 14 14 14 15 14 14 12
216122023-12-10T18:58:24.286Z20/30 Write block 434 len 16384 data: 8 8 6 7
216132023-12-10T18:58:24.286Z21/30 Read block 514 len 16384
216142023-12-10T18:58:24.289Z22/30 Read block 553 len 40960
216152023-12-10T18:58:24.292Z23/30 Read block 324 len 4096
216162023-12-10T18:58:24.292Z24/30 Write block 93 len 20480 data: 15 15 17 12 9
216172023-12-10T18:58:24.295Z25/30 Flush
216182023-12-10T18:58:24.298Z26/30 Write block 104 len 20480 data: 8 8 10 10 13
216192023-12-10T18:58:24.301Z27/30 Read block 319 len 16384
216202023-12-10T18:58:24.304Z28/30 Write block 49 len 40960 data: 11 11 10 9 8 10 9 10 10 10
216212023-12-10T18:58:24.304Z29/30 Write block 208 len 4096 data: 10
216222023-12-10T18:58:24.304Z30/30 Read block 301 len 28672
216232023-12-10T18:58:24.308Z----------------------------------------------------------------
216242023-12-10T18:58:24.308Z Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:30
216252023-12-10T18:58:24.308ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
216262023-12-10T18:58:24.308Z 1 Acked 1000 Write 8 Done Sent Done false
216272023-12-10T18:58:24.308Z 2 Acked 1001 Read 5 Done Sent Done false
216282023-12-10T18:58:24.308Z 3 Acked 1002 Write 6 Done Sent Done false
216292023-12-10T18:58:24.308Z 4 Acked 1003 Write 5 Done Sent Done false
216302023-12-10T18:58:24.308Z 5 Acked 1004 Read 2 Done Sent Done false
216312023-12-10T18:58:24.308Z 6 Acked 1005 Write 9 Done Sent Done false
216322023-12-10T18:58:24.308Z 7 Acked 1006 Write 7 Done Sent Done false
216332023-12-10T18:58:24.309Z 8 Acked 1007 Write 8 Done Sent Done false
216342023-12-10T18:58:24.309Z 9 Acked 1008 Read 5 Done Sent Done false
216352023-12-10T18:58:24.309Z 10 Acked 1009 Read 5 Done Sent Done false
216362023-12-10T18:58:24.309Z 11 Acked 1010 Read 8 Done Sent Done false
216372023-12-10T18:58:24.309Z 12 Acked 1011 Write 1 Done Sent Done false
216382023-12-10T18:58:24.310Z 13 Acked 1012 Write 1 Done Sent Done false
216392023-12-10T18:58:24.310Z 14 Acked 1013 Write 7 Done Sent Done false
216402023-12-10T18:58:24.310Z 15 Acked 1014 Write 9 Done Sent Done false
216412023-12-10T18:58:24.310Z 16 Acked 1015 Write 7 Done Sent Done false
216422023-12-10T18:58:24.310Z 17 Acked 1016 Write 6 Done Sent Done false
216432023-12-10T18:58:24.310Z 18 Acked 1017 Write 8 Done Sent Done false
216442023-12-10T18:58:24.310Z 19 Acked 1018 Write 9 Done Sent Done false
216452023-12-10T18:58:24.310Z 20 Acked 1019 Write 4 Done Sent Done false
216462023-12-10T18:58:24.310Z 21 Acked 1020 Read 4 Done Sent Done false
216472023-12-10T18:58:24.310Z 22 Acked 1021 Read 10 Done Sent Done false
216482023-12-10T18:58:24.310Z 23 Acked 1022 Read 1 Done Sent Done false
216492023-12-10T18:58:24.310Z 24 Acked 1023 Write 5 Done Sent Done false
216502023-12-10T18:58:24.310Z 25 Acked 1024 Flush 0 Done Sent Done false
216512023-12-10T18:58:24.310Z 26 Acked 1025 Write 5 Done Sent Done false
216522023-12-10T18:58:24.310Z 27 Acked 1026 Read 4 Done Sent Done false
216532023-12-10T18:58:24.310Z 28 Acked 1027 Write 10 Done Sent Done false
216542023-12-10T18:58:24.310Z 29 Acked 1028 Write 1 Done Sent Done false
216552023-12-10T18:58:24.310Z 30 Acked 1029 Read 7 Done Sent Done false
216562023-12-10T18:58:24.311Z STATES DS:0 DS:1 DS:2 TOTAL
216572023-12-10T18:58:24.311Z New 0 0 0 0
216582023-12-10T18:58:24.311Z Sent 0 30 0 30
216592023-12-10T18:58:24.311Z Done 30 0 30 60
216602023-12-10T18:58:24.311Z Skipped 0 0 0 0
216612023-12-10T18:58:24.311Z Error 0 0 0 0
216622023-12-10T18:58:24.311ZLast Flush: 1024 0 1024
216632023-12-10T18:58:24.311ZDownstairs last five completed:
216642023-12-10T18:58:24.311ZUpstairs last five completed: 30 29 28 27 26 25
216652023-12-10T18:58:24.311ZWrote out file "/tmp/test_repair_verify.data"
216662023-12-10T18:58:24.315Z
216672023-12-10T18:58:24.318ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
216682023-12-10T18:58:24.342ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
216692023-12-10T18:58:24.342Z 0 000-019 175 175 175 308 308 308 F F F
216702023-12-10T18:58:24.342Z 1 020-039 175 175 175 308 308 308 F F F
216712023-12-10T18:58:24.342Z 2 040-059 169 169 169 296 296 296 T F T <---
216722023-12-10T18:58:24.342Z 3 060-079 175 175 175 308 308 308 F F F
216732023-12-10T18:58:24.342Z 4 080-099 176 174 176 309 305 309 F F F <---
216742023-12-10T18:58:24.343Z 5 100-119 176 175 176 309 308 309 T F T <---
216752023-12-10T18:58:24.345Z 6 120-139 175 175 175 308 308 308 F F F
216762023-12-10T18:58:24.346Z 7 140-159 175 175 175 308 308 308 F F F
216772023-12-10T18:58:24.346Z 8 160-179 172 172 172 302 302 302 F F F
216782023-12-10T18:58:24.346Z 9 180-199 176 173 176 309 304 309 F F F <---
216792023-12-10T18:58:24.346Z 10 200-219 171 171 171 300 300 300 T F T <---
216802023-12-10T18:58:24.346Z 11 220-239 171 171 171 300 300 300 F F F
216812023-12-10T18:58:24.346Z 12 240-259 176 172 176 309 301 309 F F F <---
216822023-12-10T18:58:24.346Z 13 260-279 176 174 176 309 307 309 F F F <---
216832023-12-10T18:58:24.347Z 14 280-299 170 170 170 299 299 299 F F F
216842023-12-10T18:58:24.347Z 15 300-319 170 170 170 299 299 299 F F F
216852023-12-10T18:58:24.347Z 16 320-339 171 171 171 300 300 300 F F F
216862023-12-10T18:58:24.347Z 17 340-359 175 175 175 308 308 308 F F F
216872023-12-10T18:58:24.347Z 18 360-379 176 172 176 309 301 309 F F F <---
216882023-12-10T18:58:24.347Z 19 380-399 176 172 176 309 301 309 F F F <---
216892023-12-10T18:58:24.347Z 20 400-419 176 175 176 309 308 309 F F F <---
216902023-12-10T18:58:24.347Z 21 420-439 176 171 176 309 300 309 F F F <---
216912023-12-10T18:58:24.347Z 22 440-459 176 162 176 309 283 309 F F F <---
216922023-12-10T18:58:24.347Z 23 460-479 176 173 176 309 304 309 F F F <---
216932023-12-10T18:58:24.347Z 24 480-499 176 170 176 309 299 309 F F F <---
216942023-12-10T18:58:24.347Z 25 500-519 174 174 174 305 305 305 F F F
216952023-12-10T18:58:24.347Z 26 520-539 176 170 176 309 298 309 F F F <---
216962023-12-10T18:58:24.347Z 27 540-559 174 174 174 307 307 307 F F F
216972023-12-10T18:58:24.347Z 28 560-579 176 175 176 309 308 309 F F F <---
216982023-12-10T18:58:24.347Z 29 580-599 164 164 164 286 286 286 F F F
216992023-12-10T18:58:24.347ZMax gen: 176, Max flush: 309
217002023-12-10T18:58:24.347ZError: Difference in extent metadata found!
217012023-12-10T18:58:24.351ZA Difference in extent metadata is expected here
217022023-12-10T18:58:24.351ZOn loop 88
217032023-12-10T18:58:24.351Z
217042023-12-10T18:58:24.351ZVerifying data now
217052023-12-10T18:58:29.842ZNo change after verify
217062023-12-10T18:58:29.842ZLoop: 88 Downstairs dump after verify (and repair):
217072023-12-10T18:58:29.862ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
217082023-12-10T18:58:29.865Z 0 000-019 175 175 175 308 308 308 F F F
217092023-12-10T18:58:29.865Z 1 020-039 175 175 175 308 308 308 F F F
217102023-12-10T18:58:29.865Z 2 040-059 177 177 177 310 310 310 F F F
217112023-12-10T18:58:29.865Z 3 060-079 175 175 175 308 308 308 F F F
217122023-12-10T18:58:29.869Z 4 080-099 176 176 176 309 309 309 F F F
217132023-12-10T18:58:29.869Z 5 100-119 177 177 177 310 310 310 F F F
217142023-12-10T18:58:29.869Z 6 120-139 175 175 175 308 308 308 F F F
217152023-12-10T18:58:29.869Z 7 140-159 175 175 175 308 308 308 F F F
217162023-12-10T18:58:29.869Z 8 160-179 172 172 172 302 302 302 F F F
217172023-12-10T18:58:29.869Z 9 180-199 176 176 176 309 309 309 F F F
217182023-12-10T18:58:29.869Z 10 200-219 177 177 177 310 310 310 F F F
217192023-12-10T18:58:29.869Z 11 220-239 171 171 171 300 300 300 F F F
217202023-12-10T18:58:29.869Z 12 240-259 176 176 176 309 309 309 F F F
217212023-12-10T18:58:29.869Z 13 260-279 176 176 176 309 309 309 F F F
217222023-12-10T18:58:29.869Z 14 280-299 170 170 170 299 299 299 F F F
217232023-12-10T18:58:29.869Z 15 300-319 170 170 170 299 299 299 F F F
217242023-12-10T18:58:29.869Z 16 320-339 171 171 171 300 300 300 F F F
217252023-12-10T18:58:29.869Z 17 340-359 175 175 175 308 308 308 F F F
217262023-12-10T18:58:29.869Z 18 360-379 176 176 176 309 309 309 F F F
217272023-12-10T18:58:29.869Z 19 380-399 176 176 176 309 309 309 F F F
217282023-12-10T18:58:29.870Z 20 400-419 176 176 176 309 309 309 F F F
217292023-12-10T18:58:29.870Z 21 420-439 176 176 176 309 309 309 F F F
217302023-12-10T18:58:29.870Z 22 440-459 176 176 176 309 309 309 F F F
217312023-12-10T18:58:29.870Z 23 460-479 176 176 176 309 309 309 F F F
217322023-12-10T18:58:29.870Z 24 480-499 176 176 176 309 309 309 F F F
217332023-12-10T18:58:29.870Z 25 500-519 174 174 174 305 305 305 F F F
217342023-12-10T18:58:29.870Z 26 520-539 176 176 176 309 309 309 F F F
217352023-12-10T18:58:29.870Z 27 540-559 174 174 174 307 307 307 F F F
217362023-12-10T18:58:29.870Z 28 560-579 176 176 176 309 309 309 F F F
217372023-12-10T18:58:29.870Z 29 580-599 164 164 164 286 286 286 F F F
217382023-12-10T18:58:29.870ZMax gen: 177, Max flush: 310
217392023-12-10T18:58:29.870Z
217402023-12-10T18:58:29.870ZBegin loop 89
217412023-12-10T18:58:29.870ZDownstairs to restart: 2
217422023-12-10T18:58:29.898Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.89649725Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874}
217432023-12-10T18:58:29.903Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.89694854Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874}
217442023-12-10T18:58:29.903Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.897061154Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874}
217452023-12-10T18:58:29.903Z{"msg":"Crucible 4cc694bb-b6e9-477e-aef9-8209ff184fc4 has session id: c67da613-8db4-49bb-9603-7481323f6ea7","v":Crucible runtime is spawned
217462023-12-10T18:58:29.903Z0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.897138544Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217472023-12-10T18:58:29.904Z{"msg":"Upstairs opts: Upstairs UUID: 4cc694bb-b6e9-477e-aef9-8209ff184fc4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","The guest has requested activation with gen:178
217482023-12-10T18:58:29.904Zv":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.897270934Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217492023-12-10T18:58:29.905Z{"msg":"Crucible stats registered with UUID: 4cc694bb-b6e9-477e-aef9-8209ff184fc4","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.897334291Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217502023-12-10T18:58:29.905Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.897501683Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7","task":"up_listen"}
217512023-12-10T18:58:29.905Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.897546286Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217522023-12-10T18:58:29.905Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.897578751Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217532023-12-10T18:58:29.906Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.897607368Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"looper":"2","session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217542023-12-10T18:58:29.907Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.897653569Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"looper":"1","session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217552023-12-10T18:58:29.907Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.897693156Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"looper":"0","session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217562023-12-10T18:58:29.907Z{"msg":"[0] 4cc694bb-b6e9-477e-aef9-8209ff184fc4 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.897796372Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"looper":"0","session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217572023-12-10T18:58:29.907Z{"msg":"[1] 4cc694bb-b6e9-477e-aef9-8209ff184fc4 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.897843588Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"looper":"1","session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217582023-12-10T18:58:29.907Z{"msg":"4cc694bb-b6e9-477e-aef9-8209ff184fc4 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.89791051Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217592023-12-10T18:58:29.907Z{"msg":"Set desired generation to :178","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.898001087Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217602023-12-10T18:58:29.907Z{"msg":"[2] 4cc694bb-b6e9-477e-aef9-8209ff184fc4 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.898039628Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"looper":"2","session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217612023-12-10T18:58:29.907Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.898090867Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217622023-12-10T18:58:29.907Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.898414066Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217632023-12-10T18:58:29.907Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.89850285Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217642023-12-10T18:58:29.907Z{"msg":"[1] 4cc694bb-b6e9-477e-aef9-8209ff184fc4 (c67da613-8db4-49bb-9603-7481323f6ea7) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.898787279Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217652023-12-10T18:58:29.907Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.898898868Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217662023-12-10T18:58:29.907Z{"msg":"[1] client is_active_req TRUE, promote! session c67da613-8db4-49bb-9603-7481323f6ea7","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.898947026Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217672023-12-10T18:58:29.907Z{"msg":"[0] 4cc694bb-b6e9-477e-aef9-8209ff184fc4 (c67da613-8db4-49bb-9603-7481323f6ea7) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.899018918Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217682023-12-10T18:58:29.908Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.899056776Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217692023-12-10T18:58:29.908Z{"msg":"[0] client is_active_req TRUE, promote! session c67da613-8db4-49bb-9603-7481323f6ea7","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.899096026Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217702023-12-10T18:58:29.908Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.899632236Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217712023-12-10T18:58:29.908Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.89974436Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217722023-12-10T18:58:29.908Z{"msg":"4cc694bb-b6e9-477e-aef9-8209ff184fc4 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.89978395Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217732023-12-10T18:58:29.908Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.900012417Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217742023-12-10T18:58:29.908Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.900129818Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217752023-12-10T18:58:29.914Z{"msg":"4cc694bb-b6e9-477e-aef9-8209ff184fc4 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.900281601Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217762023-12-10T18:58:29.914Z{"msg":"[2] 4cc694bb-b6e9-477e-aef9-8209ff184fc4 (c67da613-8db4-49bb-9603-7481323f6ea7) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.900371545Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217772023-12-10T18:58:29.914Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.900448313Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217782023-12-10T18:58:29.914Z{"msg":"[2] client is_active_req TRUE, promote! session c67da613-8db4-49bb-9603-7481323f6ea7","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.900496215Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217792023-12-10T18:58:29.914Z{"msg":"[1] 4cc694bb-b6e9-477e-aef9-8209ff184fc4 (c67da613-8db4-49bb-9603-7481323f6ea7) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.900596829Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217802023-12-10T18:58:29.914Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.900623386Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217812023-12-10T18:58:29.914Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:29.900643714Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217822023-12-10T18:58:29.914Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.900685575Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217832023-12-10T18:58:29.915Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.900709145Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217842023-12-10T18:58:29.915Z{"msg":"4cc694bb-b6e9-477e-aef9-8209ff184fc4 WaitActive WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.900730657Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217852023-12-10T18:58:29.915Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.900809632Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217862023-12-10T18:58:29.915Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.900832129Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217872023-12-10T18:58:29.915Z{"msg":"[0] 4cc694bb-b6e9-477e-aef9-8209ff184fc4 (c67da613-8db4-49bb-9603-7481323f6ea7) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.90098074Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217882023-12-10T18:58:29.915Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.901035537Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217892023-12-10T18:58:29.915Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:29.901060068Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217902023-12-10T18:58:29.915Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.90118619Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217912023-12-10T18:58:29.915Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.901225032Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217922023-12-10T18:58:29.915Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.901292299Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217932023-12-10T18:58:29.915Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.901354807Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217942023-12-10T18:58:29.915Z{"msg":"4cc694bb-b6e9-477e-aef9-8209ff184fc4 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.901389897Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217952023-12-10T18:58:29.915Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.901458204Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217962023-12-10T18:58:29.915Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.9014831Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217972023-12-10T18:58:29.915Z{"msg":"4cc694bb-b6e9-477e-aef9-8209ff184fc4 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.901504547Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217982023-12-10T18:58:29.915Z{"msg":"[2] 4cc694bb-b6e9-477e-aef9-8209ff184fc4 (c67da613-8db4-49bb-9603-7481323f6ea7) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.901746877Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
217992023-12-10T18:58:29.916Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.901801549Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218002023-12-10T18:58:29.916Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:29.901835877Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218012023-12-10T18:58:29.916Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.901875243Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218022023-12-10T18:58:29.916Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.901934544Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218032023-12-10T18:58:29.916Z{"msg":"4cc694bb-b6e9-477e-aef9-8209ff184fc4 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.90196048Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218042023-12-10T18:58:29.916Z{"msg":"[0]R flush_numbers[0..12]: [308, 308, 310, 308, 309, 310, 308, 308, 302, 309, 310, 300]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.902003945Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218052023-12-10T18:58:29.916Z{"msg":"[0]R generation[0..12]: [175, 175, 177, 175, 176, 177, 175, 175, 172, 176, 177, 171]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.902027864Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218062023-12-10T18:58:29.916Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.902120893Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218072023-12-10T18:58:29.916Z{"msg":"[1]R flush_numbers[0..12]: [308, 308, 310, 308, 309, 310, 308, 308, 302, 309, 310, 300]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.902157653Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218082023-12-10T18:58:29.916Z{"msg":"[1]R generation[0..12]: [175, 175, 177, 175, 176, 177, 175, 175, 172, 176, 177, 171]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:178
218092023-12-10T18:58:29.916Z,"time":"2023-12-10T18:58:29.902182444Z","hostname":"Wait for a query_work_queue command to finish before sending IO
218102023-12-10T18:58:29.916Zip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218112023-12-10T18:58:29.916Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.902207107Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218122023-12-10T18:58:29.916Z{"msg":"[2]R flush_numbers[0..12]: [308, 308, 310, 308, 309, 310, 308, 308, 302, 309, 310, 300]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.90222882Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218132023-12-10T18:58:29.916Z{"msg":"[2]R generation[0..12]: [175, 175, 177, 175, 176, 177, 175, 175, 172, 176, 177, 171]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.902291001Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218142023-12-10T18:58:29.916Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.902335242Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218152023-12-10T18:58:29.916Z{"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.902401444Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218162023-12-10T18:58:29.917Z{"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.902472586Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218172023-12-10T18:58:29.917Z{"msg":"Next flush: 311","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.902581752Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218182023-12-10T18:58:29.917Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.902624331Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218192023-12-10T18:58:29.917Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.902681192Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218202023-12-10T18:58:29.917Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.902720762Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218212023-12-10T18:58:29.918Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.90277985Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218222023-12-10T18:58:29.918Z{"msg":"4cc694bb-b6e9-477e-aef9-8209ff184fc4 is now active with session: c67da613-8db4-49bb-9603-7481323f6ea7","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.902822293Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218232023-12-10T18:58:29.918Z{"msg":"4cc694bb-b6e9-477e-aef9-8209ff184fc4 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.902896161Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218242023-12-10T18:58:29.918Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.902936456Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218252023-12-10T18:58:29.918Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.903003367Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218262023-12-10T18:58:29.918Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.903042443Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
218272023-12-10T18:58:29.918Zc67da613-8db4-49bb-9603-7481323f6ea7"}
218282023-12-10T18:58:29.918Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.903148788Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218292023-12-10T18:58:29.918Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.903192445Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218302023-12-10T18:58:29.918Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.903252868Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218312023-12-10T18:58:29.919Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.903292595Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218322023-12-10T18:58:29.919Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.903347594Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218332023-12-10T18:58:29.919Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.903387004Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218342023-12-10T18:58:29.919Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.903443157Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218352023-12-10T18:58:29.919Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:29.903481919Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1874,"session_id":"c67da613-8db4-49bb-9603-7481323f6ea7"}
218362023-12-10T18:58:29.919ZLoading write count information from file "/tmp/test_repair_verify.data"
218372023-12-10T18:58:29.919ZRun Repair workload
218382023-12-10T18:58:29.919Z01/30 Write block 74 len 12288 data: 18 16 15
218392023-12-10T18:58:29.919Z02/30 Read block 126 len 40960
218402023-12-10T18:58:29.919Z03/30 Read block 565 len 12288
218412023-12-10T18:58:29.923Z04/30 Write block 450 len 12288 data: 15 16 15
218422023-12-10T18:58:29.923Z05/30 Write block 319 len 28672 data: 15 14 14 14 14 14 12
218432023-12-10T18:58:29.923Z06/30 Write block 515 len 8192 data: 9 12
218442023-12-10T18:58:29.923Z07/30 Write block 295 len 28672 data: 11 16 16 16 18 14 16
218452023-12-10T18:58:29.923Z08/30 Write block 13 len 24576 data: 18 20 19 16 18 15
218462023-12-10T18:58:29.923Z09/30 Read block 20 len 4096
218472023-12-10T18:58:29.926Z10/30 Flush
218482023-12-10T18:58:29.926Z11/30 Write block 429 len 40960 data: 11 11 10 9 9 9 9 7 8 9
218492023-12-10T18:58:29.930Z12/30 Read block 345 len 28672
218502023-12-10T18:58:29.930Z13/30 Write block 124 len 20480 data: 16 15 16 15 16
218512023-12-10T18:58:29.930Z14/30 Flush
218522023-12-10T18:58:29.933Z15/30 Write block 546 len 28672 data: 13 12 12 10 9 7 7
218532023-12-10T18:58:29.933Z16/30 Write block 400 len 4096 data: 16
218542023-12-10T18:58:29.933Z17/30 Read block 554 len 8192
218552023-12-10T18:58:29.936Z18/30 Read block 433 len 16384
218562023-12-10T18:58:29.936Z19/30 Read block 532 len 8192
218572023-12-10T18:58:29.939Z20/30 Read block 25 len 16384
218582023-12-10T18:58:29.939Z21/30 Write block 264 len 16384 data: 14 16 16 16
218592023-12-10T18:58:29.942Z22/30 Write block 170 len 20480 data: 16 17 12 12 12
218602023-12-10T18:58:29.943Z23/30 Flush
218612023-12-10T18:58:29.946Z24/30 Write block 568 len 20480 data: 13 15 17 15 13
218622023-12-10T18:58:29.946Z25/30 Flush
218632023-12-10T18:58:29.949Z26/30 Flush
218642023-12-10T18:58:29.949Z27/30 Write block 2 len 16384 data: 7 6 5 6
218652023-12-10T18:58:29.952Z28/30 Read block 208 len 28672
218662023-12-10T18:58:29.952Z29/30 Write block 144 len 16384 data: 7 7 11 11
218672023-12-10T18:58:29.956Z30/30 Read block 441 len 24576
218682023-12-10T18:58:29.956Z----------------------------------------------------------------
218692023-12-10T18:58:29.956Z Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30
218702023-12-10T18:58:29.956ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
218712023-12-10T18:58:29.956Z 1 Acked 1000 Write 3 Done Done Done false
218722023-12-10T18:58:29.956Z 2 Acked 1001 Read 10 Done Done Done false
218732023-12-10T18:58:29.956Z 3 Acked 1002 Read 3 Done Done Sent false
218742023-12-10T18:58:29.956Z 4 Acked 1003 Write 3 Done Done Sent false
218752023-12-10T18:58:29.956Z 5 Acked 1004 Write 7 Done Done Sent false
218762023-12-10T18:58:29.956Z 6 Acked 1005 Write 2 Done Done Sent false
218772023-12-10T18:58:29.956Z 7 Acked 1006 Write 7 Done Done Sent false
218782023-12-10T18:58:29.956Z 8 Acked 1007 Write 6 Done Done Sent false
218792023-12-10T18:58:29.956Z 9 Acked 1008 Read 1 Done Done Sent false
218802023-12-10T18:58:29.956Z 10 Acked 1009 Flush 0 Done Done Sent false
218812023-12-10T18:58:29.956Z 11 Acked 1010 Write 10 Done Done Sent false
218822023-12-10T18:58:29.956Z 12 Acked 1011 Read 7 Done Done Sent false
218832023-12-10T18:58:29.957Z 13 Acked 1012 Write 5 Done Done Sent false
218842023-12-10T18:58:29.957Z 14 Acked 1013 Flush 0 Done Done Sent false
218852023-12-10T18:58:29.957Z 15 Acked 1014 Write 7 Done Done Sent false
218862023-12-10T18:58:29.957Z 16 Acked 1015 Write 1 Done Done Sent false
218872023-12-10T18:58:29.957Z 17 Acked 1016 Read 2 Done Done Sent false
218882023-12-10T18:58:29.957Z 18 Acked 1017 Read 4 Done Done Sent false
218892023-12-10T18:58:29.957Z 19 Acked 1018 Read 2 Done Done Sent false
218902023-12-10T18:58:29.957Z 20 Acked 1019 Read 4 Done Done Sent false
218912023-12-10T18:58:29.957Z 21 Acked 1020 Write 4 Done Done Sent false
218922023-12-10T18:58:29.957Z 22 Acked 1021 Write 5 Done Done Sent false
218932023-12-10T18:58:29.957Z 23 Acked 1022 Flush 0 Done Done Sent false
218942023-12-10T18:58:29.957Z 24 Acked 1023 Write 5 Done Done Sent false
218952023-12-10T18:58:29.957Z 25 Acked 1024 Flush 0 Done Done Sent false
218962023-12-10T18:58:29.957Z 26 Acked 1025 Flush 0 Done Done Sent false
218972023-12-10T18:58:29.957Z 27 Acked 1026 Write 4 Done Done Sent false
218982023-12-10T18:58:29.958Z 28 Acked 1027 Read 7 Done Done Sent false
218992023-12-10T18:58:29.958Z 29 Acked 1028 Write 4 Done Done Sent false
219002023-12-10T18:58:29.958Z 30 Acked 1029 Read 6 Done Sent Sent false
219012023-12-10T18:58:29.958Z STATES DS:0 DS:1 DS:2 TOTAL
219022023-12-10T18:58:29.958Z New 0 0 0 0
219032023-12-10T18:58:29.959Z Sent 0 1 28 29
219042023-12-10T18:58:29.959Z Done 30 29 2 61
219052023-12-10T18:58:29.959Z Skipped 0 0 0 0
219062023-12-10T18:58:29.959Z Error 0 0 0 0
219072023-12-10T18:58:29.959ZLast Flush: 1025 1025 0
219082023-12-10T18:58:29.959ZDownstairs last five completed:
219092023-12-10T18:58:29.959ZUpstairs last five completed: 30 29 28 27 26 25
219102023-12-10T18:58:29.962ZWrote out file "/tmp/test_repair_verify.data"
219112023-12-10T18:58:29.962Z
219122023-12-10T18:58:29.968ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
219132023-12-10T18:58:29.993ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
219142023-12-10T18:58:29.993Z 0 000-019 178 178 175 311 311 308 T T F <---
219152023-12-10T18:58:29.997Z 1 020-039 175 175 175 308 308 308 F F F
219162023-12-10T18:58:29.997Z 2 040-059 177 177 177 310 310 310 F F F
219172023-12-10T18:58:29.997Z 3 060-079 178 178 175 311 311 308 F F T <---
219182023-12-10T18:58:29.997Z 4 080-099 176 176 176 309 309 309 F F F
219192023-12-10T18:58:29.997Z 5 100-119 177 177 177 310 310 310 F F F
219202023-12-10T18:58:29.998Z 6 120-139 178 178 175 312 312 308 F F F <---
219212023-12-10T18:58:29.998Z 7 140-159 175 175 175 308 308 308 T T F <---
219222023-12-10T18:58:29.998Z 8 160-179 178 178 172 313 313 302 F F F <---
219232023-12-10T18:58:29.998Z 9 180-199 176 176 176 309 309 309 F F F
219242023-12-10T18:58:29.998Z 10 200-219 177 177 177 310 310 310 F F F
219252023-12-10T18:58:29.998Z 11 220-239 171 171 171 300 300 300 F F F
219262023-12-10T18:58:29.998Z 12 240-259 176 176 176 309 309 309 F F F
219272023-12-10T18:58:29.998Z 13 260-279 178 178 176 313 313 309 F F F <---
219282023-12-10T18:58:29.998Z 14 280-299 178 178 170 311 311 299 F F F <---
219292023-12-10T18:58:29.998Z 15 300-319 178 178 170 311 311 299 F F F <---
219302023-12-10T18:58:29.998Z 16 320-339 178 178 171 311 311 300 F F F <---
219312023-12-10T18:58:29.998Z 17 340-359 175 175 175 308 308 308 F F F
219322023-12-10T18:58:29.998Z 18 360-379 176 176 176 309 309 309 F F F
219332023-12-10T18:58:29.998Z 19 380-399 176 176 176 309 309 309 F F F
219342023-12-10T18:58:29.998Z 20 400-419 178 178 176 313 313 309 F F F <---
219352023-12-10T18:58:29.998Z 21 420-439 178 178 176 312 312 309 F F F <---
219362023-12-10T18:58:29.998Z 22 440-459 178 178 176 311 311 309 F F F <---
219372023-12-10T18:58:29.999Z 23 460-479 176 176 176 309 309 309 F F F
219382023-12-10T18:58:29.999Z 24 480-499 176 176 176 309 309 309 F F F
219392023-12-10T18:58:29.999Z 25 500-519 178 178 174 311 311 305 F F F <---
219402023-12-10T18:58:29.999Z 26 520-539 176 176 176 309 309 309 F F F
219412023-12-10T18:58:29.999Z 27 540-559 178 178 174 313 313 307 F F F <---
219422023-12-10T18:58:29.999Z 28 560-579 178 178 176 314 314 309 F F F <---
219432023-12-10T18:58:29.999Z 29 580-599 164 164 164 286 286 286 F F F
219442023-12-10T18:58:29.999ZMax gen: 178, Max flush: 314
219452023-12-10T18:58:29.999ZError: Difference in extent metadata found!
219462023-12-10T18:58:29.999ZA Difference in extent metadata is expected here
219472023-12-10T18:58:29.999ZOn loop 89
219482023-12-10T18:58:29.999Z
219492023-12-10T18:58:30.003ZVerifying data now
219502023-12-10T18:58:35.408ZNo change after verify
219512023-12-10T18:58:35.408ZLoop: 89 Downstairs dump after verify (and repair):
219522023-12-10T18:58:35.431ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
219532023-12-10T18:58:35.431Z 0 000-019 179 179 179 315 315 315 F F F
219542023-12-10T18:58:35.434Z 1 020-039 175 175 175 308 308 308 F F F
219552023-12-10T18:58:35.434Z 2 040-059 177 177 177 310 310 310 F F F
219562023-12-10T18:58:35.434Z 3 060-079 178 178 178 311 311 311 F F F
219572023-12-10T18:58:35.434Z 4 080-099 176 176 176 309 309 309 F F F
219582023-12-10T18:58:35.434Z 5 100-119 177 177 177 310 310 310 F F F
219592023-12-10T18:58:35.434Z 6 120-139 178 178 178 312 312 312 F F F
219602023-12-10T18:58:35.434Z 7 140-159 179 179 179 315 315 315 F F F
219612023-12-10T18:58:35.435Z 8 160-179 178 178 178 313 313 313 F F F
219622023-12-10T18:58:35.435Z 9 180-199 176 176 176 309 309 309 F F F
219632023-12-10T18:58:35.435Z 10 200-219 177 177 177 310 310 310 F F F
219642023-12-10T18:58:35.435Z 11 220-239 171 171 171 300 300 300 F F F
219652023-12-10T18:58:35.435Z 12 240-259 176 176 176 309 309 309 F F F
219662023-12-10T18:58:35.435Z 13 260-279 178 178 178 313 313 313 F F F
219672023-12-10T18:58:35.435Z 14 280-299 178 178 178 311 311 311 F F F
219682023-12-10T18:58:35.435Z 15 300-319 178 178 178 311 311 311 F F F
219692023-12-10T18:58:35.435Z 16 320-339 178 178 178 311 311 311 F F F
219702023-12-10T18:58:35.435Z 17 340-359 175 175 175 308 308 308 F F F
219712023-12-10T18:58:35.435Z 18 360-379 176 176 176 309 309 309 F F F
219722023-12-10T18:58:35.435Z 19 380-399 176 176 176 309 309 309 F F F
219732023-12-10T18:58:35.435Z 20 400-419 178 178 178 313 313 313 F F F
219742023-12-10T18:58:35.435Z 21 420-439 178 178 178 312 312 312 F F F
219752023-12-10T18:58:35.435Z 22 440-459 178 178 178 311 311 311 F F F
219762023-12-10T18:58:35.435Z 23 460-479 176 176 176 309 309 309 F F F
219772023-12-10T18:58:35.435Z 24 480-499 176 176 176 309 309 309 F F F
219782023-12-10T18:58:35.436Z 25 500-519 178 178 178 311 311 311 F F F
219792023-12-10T18:58:35.436Z 26 520-539 176 176 176 309 309 309 F F F
219802023-12-10T18:58:35.436Z 27 540-559 178 178 178 313 313 313 F F F
219812023-12-10T18:58:35.436Z 28 560-579 178 178 178 314 314 314 F F F
219822023-12-10T18:58:35.436Z 29 580-599 164 164 164 286 286 286 F F F
219832023-12-10T18:58:35.436ZMax gen: 179, Max flush: 315
219842023-12-10T18:58:35.440Z
219852023-12-10T18:58:35.440ZBegin loop 90
219862023-12-10T18:58:35.440ZDownstairs to restart: 2
219872023-12-10T18:58:35.465Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.462924516Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882}
219882023-12-10T18:58:35.465Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.463343094Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882}
219892023-12-10T18:58:35.465Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.463409819Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882}
219902023-12-10T18:58:35.478Z{"msg":"Crucible 33ee8af0-cd7f-42a2-a707-32b463d992ff has session id: b8f7a496-3c1f-4ec4-a463-9468dab41910","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.463433435Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
219912023-12-10T18:58:35.478Z{"msg":"Upstairs opts: Upstairs UUID: 33ee8af0-cd7f-42a2-a707-32b463d992ff, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.463469604Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
219922023-12-10T18:58:35.478Z{"msg":"Crucible stats registered with UUID: 33ee8af0-cd7f-42a2-a707-32b463d992ff","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.463504278Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
219932023-12-10T18:58:35.478ZCrucible runtime is spawned
219942023-12-10T18:58:35.478ZThe guest has requested activation with gen:180
219952023-12-10T18:58:35.479Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.46381078Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910","task":"up_listen"}
219962023-12-10T18:58:35.479Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.463885669Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"looper":"0","session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
219972023-12-10T18:58:35.479Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.463932287Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
219982023-12-10T18:58:35.480Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.463989685Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
219992023-12-10T18:58:35.480Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.464114155Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"looper":"2","session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220002023-12-10T18:58:35.480Z{"msg":"[0] 33ee8af0-cd7f-42a2-a707-32b463d992ff looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.464184229Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"looper":"0","session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220012023-12-10T18:58:35.480Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.464229127Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"looper":"1","session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220022023-12-10T18:58:35.480Z{"msg":"[2] 33ee8af0-cd7f-42a2-a707-32b463d992ff looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.464407255Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"looper":"2","session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220032023-12-10T18:58:35.480Z{"msg":"[1] 33ee8af0-cd7f-42a2-a707-32b463d992ff looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.464532994Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"looper":"1","session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220042023-12-10T18:58:35.480Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.464885157Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220052023-12-10T18:58:35.480Z{"msg":"33ee8af0-cd7f-42a2-a707-32b463d992ff active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.464972672Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220062023-12-10T18:58:35.481Z{"msg":"Set desired generation to :180","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.465137539Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220072023-12-10T18:58:35.481Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.465335767Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220082023-12-10T18:58:35.481Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.465421437Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220092023-12-10T18:58:35.481Z{"msg":"[0] 33ee8af0-cd7f-42a2-a707-32b463d992ff (b8f7a496-3c1f-4ec4-a463-9468dab41910) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.465726554Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220102023-12-10T18:58:35.481Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.465807071Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220112023-12-10T18:58:35.481Z{"msg":"[0] client is_active_req TRUE, promote! session b8f7a496-3c1f-4ec4-a463-9468dab41910","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.46583608Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220122023-12-10T18:58:35.481Z{"msg":"[1] 33ee8af0-cd7f-42a2-a707-32b463d992ff (b8f7a496-3c1f-4ec4-a463-9468dab41910) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.465880581Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220132023-12-10T18:58:35.481Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.465934715Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220142023-12-10T18:58:35.481Z{"msg":"[1] client is_active_req TRUE, promote! session b8f7a496-3c1f-4ec4-a463-9468dab41910","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.465959128Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220152023-12-10T18:58:35.481Z{"msg":"[2] 33ee8af0-cd7f-42a2-a707-32b463d992ff (b8f7a496-3c1f-4ec4-a463-9468dab41910) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.466454361Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220162023-12-10T18:58:35.481Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.466556474Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220172023-12-10T18:58:35.481Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.466601875Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220182023-12-10T18:58:35.482Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.466665774Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220192023-12-10T18:58:35.482Z{"msg":"[2] client is_active_req TRUE, promote! session b8f7a496-3c1f-4ec4-a463-9468dab41910","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.466707887Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220202023-12-10T18:58:35.482Z{"msg":"33ee8af0-cd7f-42a2-a707-32b463d992ff WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.466764413Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220212023-12-10T18:58:35.482Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.466806684Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220222023-12-10T18:58:35.482Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.466869784Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220232023-12-10T18:58:35.482Z{"msg":"33ee8af0-cd7f-42a2-a707-32b463d992ff WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.466911055Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220242023-12-10T18:58:35.482Z{"msg":"[1] 33ee8af0-cd7f-42a2-a707-32b463d992ff (b8f7a496-3c1f-4ec4-a463-9468dab41910) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.467272311Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220252023-12-10T18:58:35.482Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.46736807Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220262023-12-10T18:58:35.482Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:35.467430148Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220272023-12-10T18:58:35.482Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.467455554Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220282023-12-10T18:58:35.482Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.467505718Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220292023-12-10T18:58:35.482Z{"msg":"33ee8af0-cd7f-42a2-a707-32b463d992ff WaitActive WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.467529072Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220302023-12-10T18:58:35.482Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.467625256Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220312023-12-10T18:58:35.482Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.467648019Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220322023-12-10T18:58:35.482Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.46769766Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220332023-12-10T18:58:35.483Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.467725275Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220342023-12-10T18:58:35.483Z{"msg":"[0] 33ee8af0-cd7f-42a2-a707-32b463d992ff (b8f7a496-3c1f-4ec4-a463-9468dab41910) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.467782847Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220352023-12-10T18:58:35.484Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.467809916Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220362023-12-10T18:58:35.484Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:35.467834874Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220372023-12-10T18:58:35.484Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.467886762Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220382023-12-10T18:58:35.484Z{"msg":"33ee8af0-cd7f-42a2-a707-32b463d992ff WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.467910199Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220392023-12-10T18:58:35.485Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.467930257Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220402023-12-10T18:58:35.485Z{"msg":"33ee8af0-cd7f-42a2-a707-32b463d992ff WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.467971375Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220412023-12-10T18:58:35.485Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.467994176Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220422023-12-10T18:58:35.485Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.468034Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220432023-12-10T18:58:35.485Z{"msg":"[2] 33ee8af0-cd7f-42a2-a707-32b463d992ff (b8f7a496-3c1f-4ec4-a463-9468dab41910) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.468345771Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220442023-12-10T18:58:35.485Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.46841946Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220452023-12-10T18:58:35.485Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:35.468446527Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220462023-12-10T18:58:35.485Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.468468719Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220472023-12-10T18:58:35.485Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.468489433Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220482023-12-10T18:58:35.485Z{"msg":"33ee8af0-cd7f-42a2-a707-32b463d992ff WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.468531339Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220492023-12-10T18:58:35.485Z{"msg":"[0]R flush_numbers[0..12]: [315, 308, 310, 311, 309, 310, 312, 315, 313, 309, 310, 300]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.468555272Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220502023-12-10T18:58:35.485Z{"msg":"[0]R generation[0..12]: [179, 175, 177, 178, 176, 177, 178, 179, 178, 176, 177, 171]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.468596993Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220512023-12-10T18:58:35.485Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.46862113Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220522023-12-10T18:58:35.485Z{"msg":"[1]R flush_numbers[0..12]: [315, 308, 310, 311, 309, 310, 312, 315, 313, 309, 310, 300]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.468644274Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220532023-12-10T18:58:35.485Z{"msg":"The guest has finished waiting for activation with:180
220542023-12-10T18:58:35.485Z[1]R generation[0..12]: [179, 175, 177, 178, 176, 177, 178, 179, 178, 176, 177, 171]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
220552023-12-10T18:58:35.485Z,"time":"2023-12-10T18:58:35.468668889Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220562023-12-10T18:58:35.486Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.468690871Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220572023-12-10T18:58:35.486Z{"msg":"[2]R flush_numbers[0..12]: [315, 308, 310, 311, 309, 310, 312, 315, 313, 309, 310, 300]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.4687122Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220582023-12-10T18:58:35.487Z{"msg":"[2]R generation[0..12]: [179, 175, 177, 178, 176, 177, 178, 179, 178, 176, 177, 171]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.468733532Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220592023-12-10T18:58:35.487Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.468816675Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220602023-12-10T18:58:35.487Z{"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.468843558Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220612023-12-10T18:58:35.487Z{"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.46886369Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220622023-12-10T18:58:35.487Z{"msg":"Next flush: 316","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.468883146Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220632023-12-10T18:58:35.487Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.468917623Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220642023-12-10T18:58:35.487Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.468948863Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220652023-12-10T18:58:35.487Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.468984768Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220662023-12-10T18:58:35.487Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.469018415Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220672023-12-10T18:58:35.488Z{"msg":"33ee8af0-cd7f-42a2-a707-32b463d992ff is now active with session: b8f7a496-3c1f-4ec4-a463-9468dab41910","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.469060695Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220682023-12-10T18:58:35.488Z{"msg":"33ee8af0-cd7f-42a2-a707-32b463d992ff Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.469097728Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220692023-12-10T18:58:35.488Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.469128397Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220702023-12-10T18:58:35.488Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.469177578Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220712023-12-10T18:58:35.488Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.469202409Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220722023-12-10T18:58:35.488Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.469226779Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220732023-12-10T18:58:35.488Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.469289799Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220742023-12-10T18:58:35.488Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.469315043Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220752023-12-10T18:58:35.488Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.469371862Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220762023-12-10T18:58:35.488Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.469400742Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220772023-12-10T18:58:35.488Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
220782023-12-10T18:58:35.488Z,"time":"2023-12-10T18:58:35.4694915Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220792023-12-10T18:58:35.488Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.469604291Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220802023-12-10T18:58:35.488Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:35.469630501Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1882,"session_id":"b8f7a496-3c1f-4ec4-a463-9468dab41910"}
220812023-12-10T18:58:35.488ZLoading write count information from file "/tmp/test_repair_verify.data"
220822023-12-10T18:58:35.489ZRun Repair workload
220832023-12-10T18:58:35.492Z01/30 Write block 76 len 8192 data: 16 16
220842023-12-10T18:58:35.493Z02/30 Flush
220852023-12-10T18:58:35.493Z03/30 Write block 117 len 32768 data: 16 16 15 15 14 14 16 17
220862023-12-10T18:58:35.493Z04/30 Write block 385 len 4096 data: 15
220872023-12-10T18:58:35.493Z05/30 Flush
220882023-12-10T18:58:35.493Z06/30 Write block 241 len 12288 data: 20 22 19
220892023-12-10T18:58:35.493Z07/30 Read block 34 len 36864
220902023-12-10T18:58:35.493Z08/30 Read block 492 len 40960
220912023-12-10T18:58:35.493Z09/30 Write block 213 len 32768 data: 9 13 14 18 18 16 15 16
220922023-12-10T18:58:35.493Z10/30 Flush
220932023-12-10T18:58:35.493Z11/30 Flush
220942023-12-10T18:58:35.496Z12/30 Write block 119 len 16384 data: 16 16 15 15
220952023-12-10T18:58:35.496Z13/30 Write block 359 len 32768 data: 17 15 14 13 13 9 9 10
220962023-12-10T18:58:35.496Z14/30 Write block 19 len 4096 data: 17
220972023-12-10T18:58:35.496Z15/30 Read block 512 len 24576
220982023-12-10T18:58:35.499Z16/30 Write block 53 len 32768 data: 9 11 10 11 11 11 8 10
220992023-12-10T18:58:35.499Z17/30 Read block 75 len 16384
221002023-12-10T18:58:35.503Z18/30 Read block 459 len 32768
221012023-12-10T18:58:35.505Z19/30 Read block 216 len 4096
221022023-12-10T18:58:35.506Z20/30 Read block 213 len 20480
221032023-12-10T18:58:35.509Z21/30 Read block 270 len 40960
221042023-12-10T18:58:35.511Z22/30 Write block 400 len 4096 data: 17
221052023-12-10T18:58:35.512Z23/30 Write block 321 len 32768 data: 15 15 15 15 13 13 14 13
221062023-12-10T18:58:35.514Z24/30 Write block 390 len 12288 data: 13 11 12
221072023-12-10T18:58:35.515Z25/30 Write block 68 len 36864 data: 17 17 15 17 14 17 19 17 17
221082023-12-10T18:58:35.515Z26/30 Read block 270 len 40960
221092023-12-10T18:58:35.518Z27/30 Read block 174 len 16384
221102023-12-10T18:58:35.521Z28/30 Read block 484 len 40960
221112023-12-10T18:58:35.524Z29/30 Write block 312 len 8192 data: 11 12
221122023-12-10T18:58:35.524Z30/30 Read block 48 len 12288
221132023-12-10T18:58:35.528Z----------------------------------------------------------------
221142023-12-10T18:58:35.528Z Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30
221152023-12-10T18:58:35.528ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
221162023-12-10T18:58:35.528Z 1 Acked 1000 Write 2 Done Done Done false
221172023-12-10T18:58:35.528Z 2 Acked 1001 Flush 0 Done Done Sent false
221182023-12-10T18:58:35.528Z 3 Acked 1002 Write 8 Done Done Sent false
221192023-12-10T18:58:35.528Z 4 Acked 1003 Write 1 Done Done Sent false
221202023-12-10T18:58:35.528Z 5 Acked 1004 Flush 0 Done Done Sent false
221212023-12-10T18:58:35.528Z 6 Acked 1005 Write 3 Done Done Sent false
221222023-12-10T18:58:35.528Z 7 Acked 1006 Read 9 Done Done Sent false
221232023-12-10T18:58:35.528Z 8 Acked 1007 Read 10 Done Done Sent false
221242023-12-10T18:58:35.528Z 9 Acked 1008 Write 8 Done Done Sent false
221252023-12-10T18:58:35.528Z 10 Acked 1009 Flush 0 Done Done Sent false
221262023-12-10T18:58:35.529Z 11 Acked 1010 Flush 0 Done Done Sent false
221272023-12-10T18:58:35.529Z 12 Acked 1011 Write 4 Done Done Sent false
221282023-12-10T18:58:35.529Z 13 Acked 1012 Write 8 Done Done Sent false
221292023-12-10T18:58:35.529Z 14 Acked 1013 Write 1 Done Done Sent false
221302023-12-10T18:58:35.529Z 15 Acked 1014 Read 6 Done Done Sent false
221312023-12-10T18:58:35.529Z 16 Acked 1015 Write 8 Done Done Sent false
221322023-12-10T18:58:35.529Z 17 Acked 1016 Read 4 Done Done Sent false
221332023-12-10T18:58:35.530Z 18 Acked 1017 Read 8 Done Done Sent false
221342023-12-10T18:58:35.530Z 19 Acked 1018 Read 1 Done Done Sent false
221352023-12-10T18:58:35.530Z 20 Acked 1019 Read 5 Done Done Sent false
221362023-12-10T18:58:35.530Z 21 Acked 1020 Read 10 Done Done Sent false
221372023-12-10T18:58:35.530Z 22 Acked 1021 Write 1 Done Done Sent false
221382023-12-10T18:58:35.530Z 23 Acked 1022 Write 8 Done Done Sent false
221392023-12-10T18:58:35.530Z 24 Acked 1023 Write 3 Done Done Sent false
221402023-12-10T18:58:35.530Z 25 Acked 1024 Write 9 Done Done Sent false
221412023-12-10T18:58:35.530Z 26 Acked 1025 Read 10 Done Done Sent false
221422023-12-10T18:58:35.530Z 27 Acked 1026 Read 4 Done Done Sent false
221432023-12-10T18:58:35.530Z 28 Acked 1027 Read 10 Done Done Sent false
221442023-12-10T18:58:35.530Z 29 Acked 1028 Write 2 Done Done Sent false
221452023-12-10T18:58:35.530Z 30 Acked 1029 Read 3 Done Sent Sent false
221462023-12-10T18:58:35.530Z STATES DS:0 DS:1 DS:2 TOTAL
221472023-12-10T18:58:35.530Z New 0 0 0 0
221482023-12-10T18:58:35.531Z Sent 0 1 29 30
221492023-12-10T18:58:35.531Z Done 30 29 1 60
221502023-12-10T18:58:35.531Z Skipped 0 0 0 0
221512023-12-10T18:58:35.531Z Error 0 0 0 0
221522023-12-10T18:58:35.531ZLast Flush: 1010 1010 0
221532023-12-10T18:58:35.531ZDownstairs last five completed:
221542023-12-10T18:58:35.531ZUpstairs last five completed: 30 29 28 27 26 25
221552023-12-10T18:58:35.531ZWrote out file "/tmp/test_repair_verify.data"
221562023-12-10T18:58:35.535Z
221572023-12-10T18:58:35.539ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
221582023-12-10T18:58:35.569ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
221592023-12-10T18:58:35.569Z 0 000-019 179 179 179 315 315 315 T T F <---
221602023-12-10T18:58:35.569Z 1 020-039 175 175 175 308 308 308 F F F
221612023-12-10T18:58:35.569Z 2 040-059 177 177 177 310 310 310 T T F <---
221622023-12-10T18:58:35.569Z 3 060-079 180 180 178 316 316 311 T T T <---
221632023-12-10T18:58:35.569Z 4 080-099 176 176 176 309 309 309 F F F
221642023-12-10T18:58:35.569Z 5 100-119 180 180 177 317 317 310 T T F <---
221652023-12-10T18:58:35.569Z 6 120-139 180 180 178 317 317 312 T T F <---
221662023-12-10T18:58:35.569Z 7 140-159 179 179 179 315 315 315 F F F
221672023-12-10T18:58:35.572Z 8 160-179 178 178 178 313 313 313 F F F
221682023-12-10T18:58:35.573Z 9 180-199 176 176 176 309 309 309 F F F
221692023-12-10T18:58:35.573Z 10 200-219 180 180 177 318 318 310 F F F <---
221702023-12-10T18:58:35.573Z 11 220-239 180 180 171 318 318 300 F F F <---
221712023-12-10T18:58:35.573Z 12 240-259 180 180 176 318 318 309 F F F <---
221722023-12-10T18:58:35.574Z 13 260-279 178 178 178 313 313 313 F F F
221732023-12-10T18:58:35.574Z 14 280-299 178 178 178 311 311 311 F F F
221742023-12-10T18:58:35.574Z 15 300-319 178 178 178 311 311 311 T T F <---
221752023-12-10T18:58:35.574Z 16 320-339 178 178 178 311 311 311 T T F <---
221762023-12-10T18:58:35.574Z 17 340-359 175 175 175 308 308 308 T T F <---
221772023-12-10T18:58:35.574Z 18 360-379 176 176 176 309 309 309 T T F <---
221782023-12-10T18:58:35.574Z 19 380-399 180 180 176 317 317 309 T T F <---
221792023-12-10T18:58:35.574Z 20 400-419 178 178 178 313 313 313 T T F <---
221802023-12-10T18:58:35.574Z 21 420-439 178 178 178 312 312 312 F F F
221812023-12-10T18:58:35.574Z 22 440-459 178 178 178 311 311 311 F F F
221822023-12-10T18:58:35.574Z 23 460-479 176 176 176 309 309 309 F F F
221832023-12-10T18:58:35.574Z 24 480-499 176 176 176 309 309 309 F F F
221842023-12-10T18:58:35.574Z 25 500-519 178 178 178 311 311 311 F F F
221852023-12-10T18:58:35.574Z 26 520-539 176 176 176 309 309 309 F F F
221862023-12-10T18:58:35.574Z 27 540-559 178 178 178 313 313 313 F F F
221872023-12-10T18:58:35.574Z 28 560-579 178 178 178 314 314 314 F F F
221882023-12-10T18:58:35.574Z 29 580-599 164 164 164 286 286 286 F F F
221892023-12-10T18:58:35.574ZMax gen: 180, Max flush: 318
221902023-12-10T18:58:35.575ZError: Difference in extent metadata found!
221912023-12-10T18:58:35.578ZA Difference in extent metadata is expected here
221922023-12-10T18:58:35.578ZOn loop 90
221932023-12-10T18:58:35.578Z
221942023-12-10T18:58:35.578ZVerifying data now
221952023-12-10T18:58:40.933ZNo change after verify
221962023-12-10T18:58:40.933ZLoop: 90 Downstairs dump after verify (and repair):
221972023-12-10T18:58:40.956ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
221982023-12-10T18:58:40.956Z 0 000-019 181 181 181 319 319 319 F F F
221992023-12-10T18:58:40.959Z 1 020-039 175 175 175 308 308 308 F F F
222002023-12-10T18:58:40.960Z 2 040-059 181 181 181 319 319 319 F F F
222012023-12-10T18:58:40.960Z 3 060-079 181 181 181 319 319 319 F F F
222022023-12-10T18:58:40.960Z 4 080-099 176 176 176 309 309 309 F F F
222032023-12-10T18:58:40.960Z 5 100-119 181 181 181 319 319 319 F F F
222042023-12-10T18:58:40.960Z 6 120-139 181 181 181 319 319 319 F F F
222052023-12-10T18:58:40.960Z 7 140-159 179 179 179 315 315 315 F F F
222062023-12-10T18:58:40.960Z 8 160-179 178 178 178 313 313 313 F F F
222072023-12-10T18:58:40.960Z 9 180-199 176 176 176 309 309 309 F F F
222082023-12-10T18:58:40.960Z 10 200-219 180 180 180 318 318 318 F F F
222092023-12-10T18:58:40.960Z 11 220-239 180 180 180 318 318 318 F F F
222102023-12-10T18:58:40.960Z 12 240-259 180 180 180 318 318 318 F F F
222112023-12-10T18:58:40.960Z 13 260-279 178 178 178 313 313 313 F F F
222122023-12-10T18:58:40.960Z 14 280-299 178 178 178 311 311 311 F F F
222132023-12-10T18:58:40.960Z 15 300-319 181 181 181 319 319 319 F F F
222142023-12-10T18:58:40.960Z 16 320-339 181 181 181 319 319 319 F F F
222152023-12-10T18:58:40.961Z 17 340-359 181 181 181 319 319 319 F F F
222162023-12-10T18:58:40.961Z 18 360-379 181 181 181 319 319 319 F F F
222172023-12-10T18:58:40.961Z 19 380-399 181 181 181 319 319 319 F F F
222182023-12-10T18:58:40.961Z 20 400-419 181 181 181 319 319 319 F F F
222192023-12-10T18:58:40.967Z 21 420-439 178 178 178 312 312 312 F F F
222202023-12-10T18:58:40.967Z 22 440-459 178 178 178 311 311 311 F F F
222212023-12-10T18:58:40.967Z 23 460-479 176 176 176 309 309 309 F F F
222222023-12-10T18:58:40.967Z 24 480-499 176 176 176 309 309 309 F F F
222232023-12-10T18:58:40.967Z 25 500-519 178 178 178 311 311 311 F F F
222242023-12-10T18:58:40.967Z 26 520-539 176 176 176 309 309 309 F F F
222252023-12-10T18:58:40.967Z 27 540-559 178 178 178 313 313 313 F F F
222262023-12-10T18:58:40.967Z 28 560-579 178 178 178 314 314 314 F F F
222272023-12-10T18:58:40.967Z 29 580-599 164 164 164 286 286 286 F F F
222282023-12-10T18:58:40.967ZMax gen: 181, Max flush: 319
222292023-12-10T18:58:40.971Z
222302023-12-10T18:58:40.971ZBegin loop 91
222312023-12-10T18:58:40.971ZDownstairs to restart: 1
222322023-12-10T18:58:40.994Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.989029912Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890}
222332023-12-10T18:58:40.994Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.989499274Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890}
222342023-12-10T18:58:41.003Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.989551289Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890}
222352023-12-10T18:58:41.003Z{"msg":"Crucible 15052ed2-b73d-4627-bd39-e13d442d6628 has session id: 7a01c1a6-17b9-447b-971d-6ea7706200f4","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.98957576Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222362023-12-10T18:58:41.003Z{"msg":"Upstairs opts: Upstairs UUID: 15052ed2-b73d-4627-bd39-e13d442d6628, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.989624793Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222372023-12-10T18:58:41.003Z{"msg":"Crucible stats registered with UUID: 15052ed2-b73d-4627-bd39-e13d442d6628","v":0,"name":"crucible","level":30Crucible runtime is spawned
222382023-12-10T18:58:41.004Z,"time":"2023-12-10T18:58:40.989716782Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222392023-12-10T18:58:41.004ZThe guest has requested activation with gen:182
222402023-12-10T18:58:41.004Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.989969301Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"looper":"0","session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222412023-12-10T18:58:41.004Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.990104718Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4","task":"up_listen"}
222422023-12-10T18:58:41.004Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.990204894Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222432023-12-10T18:58:41.005Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.990246462Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222442023-12-10T18:58:41.005Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.990317057Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"looper":"2","session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222452023-12-10T18:58:41.005Z{"msg":"[0] 15052ed2-b73d-4627-bd39-e13d442d6628 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.990387774Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"looper":"0","session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222462023-12-10T18:58:41.005Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.990575094Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"looper":"1","session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222472023-12-10T18:58:41.005Z{"msg":"[2] 15052ed2-b73d-4627-bd39-e13d442d6628 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.990798184Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"looper":"2","session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222482023-12-10T18:58:41.005Z{"msg":"[1] 15052ed2-b73d-4627-bd39-e13d442d6628 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.990841037Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"looper":"1","session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222492023-12-10T18:58:41.005Z{"msg":"15052ed2-b73d-4627-bd39-e13d442d6628 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.990890061Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222502023-12-10T18:58:41.005Z{"msg":"Set desired generation to :182","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.990913297Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222512023-12-10T18:58:41.005Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.991012457Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222522023-12-10T18:58:41.005Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.991323233Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222532023-12-10T18:58:41.005Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.991394938Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222542023-12-10T18:58:41.005Z{"msg":"[0] 15052ed2-b73d-4627-bd39-e13d442d6628 (7a01c1a6-17b9-447b-971d-6ea7706200f4) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.991751829Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222552023-12-10T18:58:41.005Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.991845881Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222562023-12-10T18:58:41.005Z{"msg":"[0] client is_active_req TRUE, promote! session 7a01c1a6-17b9-447b-971d-6ea7706200f4","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.991960373Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222572023-12-10T18:58:41.005Z{"msg":"[2] 15052ed2-b73d-4627-bd39-e13d442d6628 (7a01c1a6-17b9-447b-971d-6ea7706200f4) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.992086103Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222582023-12-10T18:58:41.005Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.992132552Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222592023-12-10T18:58:41.006Z{"msg":"[2] client is_active_req TRUE, promote! session 7a01c1a6-17b9-447b-971d-6ea7706200f4","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.99220037Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222602023-12-10T18:58:41.006Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.992506729Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222612023-12-10T18:58:41.006Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.992585665Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222622023-12-10T18:58:41.006Z{"msg":"15052ed2-b73d-4627-bd39-e13d442d6628 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.992632059Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222632023-12-10T18:58:41.006Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.992726889Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222642023-12-10T18:58:41.006Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.992775676Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222652023-12-10T18:58:41.006Z{"msg":"15052ed2-b73d-4627-bd39-e13d442d6628 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.992845571Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222662023-12-10T18:58:41.006Z{"msg":"[1] 15052ed2-b73d-4627-bd39-e13d442d6628 (7a01c1a6-17b9-447b-971d-6ea7706200f4) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.99288938Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222672023-12-10T18:58:41.006Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.992957739Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222682023-12-10T18:58:41.006Z{"msg":"[1] client is_active_req TRUE, promote! session 7a01c1a6-17b9-447b-971d-6ea7706200f4","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.993030334Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222692023-12-10T18:58:41.006Z{"msg":"[0] 15052ed2-b73d-4627-bd39-e13d442d6628 (7a01c1a6-17b9-447b-971d-6ea7706200f4) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.993400182Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222702023-12-10T18:58:41.006Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.993460837Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222712023-12-10T18:58:41.006Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:40.993485815Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222722023-12-10T18:58:41.006Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.99350774Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222732023-12-10T18:58:41.006Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.993528731Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222742023-12-10T18:58:41.006Z{"msg":"15052ed2-b73d-4627-bd39-e13d442d6628 WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.993570694Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222752023-12-10T18:58:41.006Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.99359357Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222762023-12-10T18:58:41.006Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.993638742Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222772023-12-10T18:58:41.007Z{"msg":"[2] 15052ed2-b73d-4627-bd39-e13d442d6628 (7a01c1a6-17b9-447b-971d-6ea7706200f4) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.993852141Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222782023-12-10T18:58:41.007Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.993918595Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222792023-12-10T18:58:41.007Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:40.994040922Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222802023-12-10T18:58:41.007Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.994103097Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222812023-12-10T18:58:41.007Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.994128935Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222822023-12-10T18:58:41.007Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.994175211Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222832023-12-10T18:58:41.007Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.99419841Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222842023-12-10T18:58:41.007Z{"msg":"15052ed2-b73d-4627-bd39-e13d442d6628 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.994239838Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222852023-12-10T18:58:41.008Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.994262906Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222862023-12-10T18:58:41.008Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.99434281Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222872023-12-10T18:58:41.008Z{"msg":"15052ed2-b73d-4627-bd39-e13d442d6628 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.994366062Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222882023-12-10T18:58:41.008Z{"msg":"[1] 15052ed2-b73d-4627-bd39-e13d442d6628 (7a01c1a6-17b9-447b-971d-6ea7706200f4) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.994552765Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222892023-12-10T18:58:41.008Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.994603273Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222902023-12-10T18:58:41.008Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:40.994626473Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222912023-12-10T18:58:41.008Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.99464639Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222922023-12-10T18:58:41.008Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.994695459Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222932023-12-10T18:58:41.008Z{"msg":"15052ed2-b73d-4627-bd39-e13d442d6628 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.994725934Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222942023-12-10T18:58:41.008Z{"msg":"[0]R flush_numbers[0..12]: [319, 308, 319, 319, 309, 319, 319, 315, 313, 309, 318, 318]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.994770637Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222952023-12-10T18:58:41.008Z{"msg":"[0]R generation[0..12]: [181, 175, 181, 181, 176, 181, 181, 179, 178, 176, 180, 180]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.994796883Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222962023-12-10T18:58:41.008Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.994844221Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222972023-12-10T18:58:41.008Z{"msg":"[1]R flush_numbers[0..12]: [319, 308, 319, 319, 309, 319, 319, 315, 313, 309, 318, 318]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.994870523Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
222982023-12-10T18:58:41.008Z{"msg":"[1]R generation[0..12]: [181, 175, 181, 181, 176, 181, 181, 179, 178, 176, 180, 180]","v":0,"The guest has finished waiting for activation with:182
222992023-12-10T18:58:41.008Zname":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
223002023-12-10T18:58:41.008Z,"time":"2023-12-10T18:58:40.994922125Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
223012023-12-10T18:58:41.008Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.994948187Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
223022023-12-10T18:58:41.009Z{"msg":"[2]R flush_numbers[0..12]: [319, 308, 319, 319, 309, 319, 319, 315, 313, 309, 318, 318]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.9950166Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
223032023-12-10T18:58:41.009Z{"msg":"[2]R generation[0..12]: [181, 175, 181, 181, 176, 181, 181, 179, 178, 176, 180, 180]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.995119113Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
223042023-12-10T18:58:41.009Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.995165226Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
223052023-12-10T18:58:41.009Z{"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.995229196Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
223062023-12-10T18:58:41.009Z{"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.995268018Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
223072023-12-10T18:58:41.009Z{"msg":"Next flush: 320","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.995301873Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
223082023-12-10T18:58:41.009Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.995335655Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
223092023-12-10T18:58:41.009Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.995392347Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
223102023-12-10T18:58:41.009Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.995431236Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
223112023-12-10T18:58:41.009Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.9954863Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
223122023-12-10T18:58:41.010Z{"msg":"15052ed2-b73d-4627-bd39-e13d442d6628 is now active with session: 7a01c1a6-17b9-447b-971d-6ea7706200f4","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.995527055Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
223132023-12-10T18:58:41.010Z{"msg":"15052ed2-b73d-4627-bd39-e13d442d6628 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.995590549Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
223142023-12-10T18:58:41.010Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.995629619Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
223152023-12-10T18:58:41.010Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.995684748Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
223162023-12-10T18:58:41.010Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.995722868Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
223172023-12-10T18:58:41.010Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.995787946Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
223182023-12-10T18:58:41.010Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.9958262Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
223192023-12-10T18:58:41.010Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.995880106Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
223202023-12-10T18:58:41.010Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.995917402Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
223212023-12-10T18:58:41.010Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.996037112Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
223222023-12-10T18:58:41.010Zip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
223232023-12-10T18:58:41.014Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.996188064Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
223242023-12-10T18:58:41.014Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.996243289Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
223252023-12-10T18:58:41.014Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:40.996281663Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1890,"session_id":"7a01c1a6-17b9-447b-971d-6ea7706200f4"}
223262023-12-10T18:58:41.014ZLoading write count information from file "/tmp/test_repair_verify.data"
223272023-12-10T18:58:41.015ZRun Repair workload
223282023-12-10T18:58:41.015Z01/30 Write block 314 len 8192 data: 13 13
223292023-12-10T18:58:41.015Z02/30 Write block 426 len 36864 data: 15 15 11 12 12 11 10 10 10
223302023-12-10T18:58:41.015Z03/30 Write block 431 len 36864 data: 12 11 11 11 10 8 9 10 8
223312023-12-10T18:58:41.015Z04/30 Write block 498 len 32768 data: 13 10 13 15 15 16 11 12
223322023-12-10T18:58:41.015Z05/30 Write block 97 len 16384 data: 10 9 9 11
223332023-12-10T18:58:41.015Z06/30 Write block 65 len 40960 data: 18 17 15 18 18 16 18 15 18 20
223342023-12-10T18:58:41.015Z07/30 Write block 362 len 32768 data: 14 14 10 10 11 10 10 11
223352023-12-10T18:58:41.015Z08/30 Read block 443 len 20480
223362023-12-10T18:58:41.015Z09/30 Flush
223372023-12-10T18:58:41.018Z10/30 Write block 124 len 12288 data: 18 16 17
223382023-12-10T18:58:41.018Z11/30 Write block 524 len 24576 data: 15 15 17 16 16 17
223392023-12-10T18:58:41.018Z12/30 Read block 137 len 40960
223402023-12-10T18:58:41.022Z13/30 Read block 50 len 4096
223412023-12-10T18:58:41.023Z14/30 Read block 52 len 32768
223422023-12-10T18:58:41.027Z15/30 Write block 214 len 8192 data: 14 15
223432023-12-10T18:58:41.027Z16/30 Write block 276 len 20480 data: 16 18 16 13 9
223442023-12-10T18:58:41.032Z17/30 Write block 339 len 20480 data: 14 13 13 14 17
223452023-12-10T18:58:41.032Z18/30 Read block 455 len 12288
223462023-12-10T18:58:41.032Z19/30 Read block 439 len 4096
223472023-12-10T18:58:41.036Z20/30 Write block 574 len 8192 data: 15 16
223482023-12-10T18:58:41.036Z21/30 Read block 261 len 24576
223492023-12-10T18:58:41.036Z22/30 Write block 547 len 32768 data: 13 13 11 10 8 8 8 8
223502023-12-10T18:58:41.036Z23/30 Read block 581 len 28672
223512023-12-10T18:58:41.039Z24/30 Write block 482 len 8192 data: 10 11
223522023-12-10T18:58:41.039Z25/30 Write block 29 len 24576 data: 8 11 14 16 10 12
223532023-12-10T18:58:41.039Z26/30 Read block 554 len 16384
223542023-12-10T18:58:41.042Z27/30 Write block 399 len 4096 data: 15
223552023-12-10T18:58:41.043Z28/30 Read block 542 len 28672
223562023-12-10T18:58:41.045Z29/30 Write block 8 len 40960 data: 14 14 15 13 17 19 21 20 17 19
223572023-12-10T18:58:41.045Z30/30 Write block 106 len 4096 data: 11
223582023-12-10T18:58:41.052Z----------------------------------------------------------------
223592023-12-10T18:58:41.052Z Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:30
223602023-12-10T18:58:41.052ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
223612023-12-10T18:58:41.052Z 1 Acked 1000 Write 2 Done Sent Done false
223622023-12-10T18:58:41.052Z 2 Acked 1001 Write 9 Done Sent Done false
223632023-12-10T18:58:41.052Z 3 Acked 1002 Write 9 Done Sent Done false
223642023-12-10T18:58:41.052Z 4 Acked 1003 Write 8 Done Sent Done false
223652023-12-10T18:58:41.052Z 5 Acked 1004 Write 4 Done Sent Done false
223662023-12-10T18:58:41.052Z 6 Acked 1005 Write 10 Done Sent Done false
223672023-12-10T18:58:41.053Z 7 Acked 1006 Write 8 Done Sent Done false
223682023-12-10T18:58:41.054Z 8 Acked 1007 Read 5 Done Sent Done false
223692023-12-10T18:58:41.054Z 9 Acked 1008 Flush 0 Done Sent Done false
223702023-12-10T18:58:41.054Z 10 Acked 1009 Write 3 Done Sent Done false
223712023-12-10T18:58:41.054Z 11 Acked 1010 Write 6 Done Sent Done false
223722023-12-10T18:58:41.054Z 12 Acked 1011 Read 10 Done Sent Done false
223732023-12-10T18:58:41.054Z 13 Acked 1012 Read 1 Done Sent Done false
223742023-12-10T18:58:41.054Z 14 Acked 1013 Read 8 Done Sent Done false
223752023-12-10T18:58:41.054Z 15 Acked 1014 Write 2 Done Sent Done false
223762023-12-10T18:58:41.054Z 16 Acked 1015 Write 5 Done Sent Done false
223772023-12-10T18:58:41.054Z 17 Acked 1016 Write 5 Done Sent Done false
223782023-12-10T18:58:41.054Z 18 Acked 1017 Read 3 Done Sent Done false
223792023-12-10T18:58:41.054Z 19 Acked 1018 Read 1 Done Sent Done false
223802023-12-10T18:58:41.054Z 20 Acked 1019 Write 2 Done Sent Done false
223812023-12-10T18:58:41.054Z 21 Acked 1020 Read 6 Done Sent Done false
223822023-12-10T18:58:41.054Z 22 Acked 1021 Write 8 Done Sent Done false
223832023-12-10T18:58:41.054Z 23 Acked 1022 Read 7 Done Sent Done false
223842023-12-10T18:58:41.054Z 24 Acked 1023 Write 2 Done Sent Done false
223852023-12-10T18:58:41.054Z 25 Acked 1024 Write 6 Done Sent Done false
223862023-12-10T18:58:41.055Z 26 Acked 1025 Read 4 Done Sent Done false
223872023-12-10T18:58:41.055Z 27 Acked 1026 Write 1 Done Sent Done false
223882023-12-10T18:58:41.055Z 28 Acked 1027 Read 7 Done Sent Done false
223892023-12-10T18:58:41.055Z 29 Acked 1028 Write 10 Sent Sent Sent false
223902023-12-10T18:58:41.055Z 30 Acked 1029 Write 1 Sent Sent Sent false
223912023-12-10T18:58:41.055Z STATES DS:0 DS:1 DS:2 TOTAL
223922023-12-10T18:58:41.056Z New 0 0 0 0
223932023-12-10T18:58:41.056Z Sent 2 30 2 34
223942023-12-10T18:58:41.056Z Done 28 0 28 56
223952023-12-10T18:58:41.056Z Skipped 0 0 0 0
223962023-12-10T18:58:41.056Z Error 0 0 0 0
223972023-12-10T18:58:41.056ZLast Flush: 1008 0 1008
223982023-12-10T18:58:41.056ZDownstairs last five completed:
223992023-12-10T18:58:41.056ZUpstairs last five completed: 30 29 28 27 26 25
224002023-12-10T18:58:41.056ZWrote out file "/tmp/test_repair_verify.data"
224012023-12-10T18:58:41.059Z
224022023-12-10T18:58:41.062ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
224032023-12-10T18:58:41.092ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
224042023-12-10T18:58:41.092Z 0 000-019 181 181 181 319 319 319 T F T <---
224052023-12-10T18:58:41.095Z 1 020-039 175 175 175 308 308 308 T F T <---
224062023-12-10T18:58:41.095Z 2 040-059 181 181 181 319 319 319 F F F
224072023-12-10T18:58:41.095Z 3 060-079 182 181 182 320 319 320 F F F <---
224082023-12-10T18:58:41.095Z 4 080-099 182 176 182 320 309 320 F F F <---
224092023-12-10T18:58:41.095Z 5 100-119 182 181 182 320 319 320 T F T <---
224102023-12-10T18:58:41.095Z 6 120-139 181 181 181 319 319 319 T F T <---
224112023-12-10T18:58:41.095Z 7 140-159 179 179 179 315 315 315 F F F
224122023-12-10T18:58:41.096Z 8 160-179 178 178 178 313 313 313 F F F
224132023-12-10T18:58:41.096Z 9 180-199 176 176 176 309 309 309 F F F
224142023-12-10T18:58:41.096Z 10 200-219 180 180 180 318 318 318 T F T <---
224152023-12-10T18:58:41.096Z 11 220-239 180 180 180 318 318 318 F F F
224162023-12-10T18:58:41.096Z 12 240-259 180 180 180 318 318 318 F F F
224172023-12-10T18:58:41.096Z 13 260-279 178 178 178 313 313 313 T F T <---
224182023-12-10T18:58:41.096Z 14 280-299 178 178 178 311 311 311 T F T <---
224192023-12-10T18:58:41.096Z 15 300-319 182 181 182 320 319 320 F F F <---
224202023-12-10T18:58:41.096Z 16 320-339 181 181 181 319 319 319 T F T <---
224212023-12-10T18:58:41.096Z 17 340-359 181 181 181 319 319 319 T F T <---
224222023-12-10T18:58:41.096Z 18 360-379 182 181 182 320 319 320 F F F <---
224232023-12-10T18:58:41.096Z 19 380-399 181 181 181 319 319 319 T F T <---
224242023-12-10T18:58:41.096Z 20 400-419 181 181 181 319 319 319 F F F
224252023-12-10T18:58:41.096Z 21 420-439 182 178 182 320 312 320 F F F <---
224262023-12-10T18:58:41.096Z 22 440-459 178 178 178 311 311 311 F F F
224272023-12-10T18:58:41.096Z 23 460-479 176 176 176 309 309 309 F F F
224282023-12-10T18:58:41.096Z 24 480-499 182 176 182 320 309 320 T F T <---
224292023-12-10T18:58:41.096Z 25 500-519 182 178 182 320 311 320 F F F <---
224302023-12-10T18:58:41.097Z 26 520-539 176 176 176 309 309 309 T F T <---
224312023-12-10T18:58:41.097Z 27 540-559 178 178 178 313 313 313 T F T <---
224322023-12-10T18:58:41.097Z 28 560-579 178 178 178 314 314 314 T F T <---
224332023-12-10T18:58:41.097Z 29 580-599 164 164 164 286 286 286 F F F
224342023-12-10T18:58:41.097ZMax gen: 182, Max flush: 320
224352023-12-10T18:58:41.097ZError: Difference in extent metadata found!
224362023-12-10T18:58:41.100ZA Difference in extent metadata is expected here
224372023-12-10T18:58:41.100ZOn loop 91
224382023-12-10T18:58:41.100Z
224392023-12-10T18:58:41.100ZVerifying data now
224402023-12-10T18:58:46.919ZNo change after verify
224412023-12-10T18:58:46.919ZLoop: 91 Downstairs dump after verify (and repair):
224422023-12-10T18:58:46.942ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
224432023-12-10T18:58:46.942Z 0 000-019 183 183 183 321 321 321 F F F
224442023-12-10T18:58:46.945Z 1 020-039 183 183 183 321 321 321 F F F
224452023-12-10T18:58:46.946Z 2 040-059 181 181 181 319 319 319 F F F
224462023-12-10T18:58:46.946Z 3 060-079 182 182 182 320 320 320 F F F
224472023-12-10T18:58:46.946Z 4 080-099 182 182 182 320 320 320 F F F
224482023-12-10T18:58:46.946Z 5 100-119 183 183 183 321 321 321 F F F
224492023-12-10T18:58:46.946Z 6 120-139 183 183 183 321 321 321 F F F
224502023-12-10T18:58:46.946Z 7 140-159 179 179 179 315 315 315 F F F
224512023-12-10T18:58:46.946Z 8 160-179 178 178 178 313 313 313 F F F
224522023-12-10T18:58:46.946Z 9 180-199 176 176 176 309 309 309 F F F
224532023-12-10T18:58:46.946Z 10 200-219 183 183 183 321 321 321 F F F
224542023-12-10T18:58:46.946Z 11 220-239 180 180 180 318 318 318 F F F
224552023-12-10T18:58:46.946Z 12 240-259 180 180 180 318 318 318 F F F
224562023-12-10T18:58:46.946Z 13 260-279 183 183 183 321 321 321 F F F
224572023-12-10T18:58:46.946Z 14 280-299 183 183 183 321 321 321 F F F
224582023-12-10T18:58:46.946Z 15 300-319 182 182 182 320 320 320 F F F
224592023-12-10T18:58:46.947Z 16 320-339 183 183 183 321 321 321 F F F
224602023-12-10T18:58:46.947Z 17 340-359 183 183 183 321 321 321 F F F
224612023-12-10T18:58:46.947Z 18 360-379 182 182 182 320 320 320 F F F
224622023-12-10T18:58:46.947Z 19 380-399 183 183 183 321 321 321 F F F
224632023-12-10T18:58:46.947Z 20 400-419 181 181 181 319 319 319 F F F
224642023-12-10T18:58:46.947Z 21 420-439 182 182 182 320 320 320 F F F
224652023-12-10T18:58:46.947Z 22 440-459 178 178 178 311 311 311 F F F
224662023-12-10T18:58:46.947Z 23 460-479 176 176 176 309 309 309 F F F
224672023-12-10T18:58:46.947Z 24 480-499 183 183 183 321 321 321 F F F
224682023-12-10T18:58:46.947Z 25 500-519 182 182 182 320 320 320 F F F
224692023-12-10T18:58:46.947Z 26 520-539 183 183 183 321 321 321 F F F
224702023-12-10T18:58:46.948Z 27 540-559 183 183 183 321 321 321 F F F
224712023-12-10T18:58:46.948Z 28 560-579 183 183 183 321 321 321 F F F
224722023-12-10T18:58:46.948Z 29 580-599 164 164 164 286 286 286 F F F
224732023-12-10T18:58:46.948ZMax gen: 183, Max flush: 321
224742023-12-10T18:58:46.954Z
224752023-12-10T18:58:46.954ZBegin loop 92
224762023-12-10T18:58:46.954ZDownstairs to restart: 1
224772023-12-10T18:58:46.977Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.975297776Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898}
224782023-12-10T18:58:46.977Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.975727899Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898}
224792023-12-10T18:58:46.982Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.975798338Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898}
224802023-12-10T18:58:46.982Z{"msg":"Crucible 6217da6e-3637-4b2d-a9dd-192703679ddd has session id: e980a471-a93a-4811-b0c4-ce07ed6fd25d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.975861632Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
224812023-12-10T18:58:46.982Z{"msg":"Upstairs opts: Upstairs UUID: 6217da6e-3637-4b2d-a9dd-192703679ddd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.975921576Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
224822023-12-10T18:58:46.982Z{"msg":"Crucible stats registered with UUID: 6217da6e-3637-4b2d-a9dd-192703679ddd","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.975963301Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
224832023-12-10T18:58:46.982ZCrucible runtime is spawned
224842023-12-10T18:58:46.982Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.976232136Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"looper":"0","session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
224852023-12-10T18:58:46.982Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30The guest has requested activation with gen:184
224862023-12-10T18:58:46.982Z,"time":"2023-12-10T18:58:46.976356251Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d","task":"up_listen"}
224872023-12-10T18:58:46.982Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.976541052Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
224882023-12-10T18:58:46.982Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.976585913Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
224892023-12-10T18:58:46.982Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.976624057Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"looper":"2","session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
224902023-12-10T18:58:46.982Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.976722435Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"looper":"1","session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
224912023-12-10T18:58:46.982Z{"msg":"[0] 6217da6e-3637-4b2d-a9dd-192703679ddd looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.976796732Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"looper":"0","session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
224922023-12-10T18:58:46.982Z{"msg":"6217da6e-3637-4b2d-a9dd-192703679ddd active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.976867994Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
224932023-12-10T18:58:46.984Z{"msg":"Set desired generation to :184","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.97693674Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
224942023-12-10T18:58:46.984Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.976977055Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
224952023-12-10T18:58:46.984Z{"msg":"[1] 6217da6e-3637-4b2d-a9dd-192703679ddd looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.977062134Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"looper":"1","session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
224962023-12-10T18:58:46.984Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.977264925Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
224972023-12-10T18:58:46.984Z{"msg":"[2] 6217da6e-3637-4b2d-a9dd-192703679ddd looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.977345576Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"looper":"2","session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
224982023-12-10T18:58:46.986Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.9774223Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
224992023-12-10T18:58:46.986Z{"msg":"[0] 6217da6e-3637-4b2d-a9dd-192703679ddd (e980a471-a93a-4811-b0c4-ce07ed6fd25d) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.97759467Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225002023-12-10T18:58:46.986Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.977710634Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225012023-12-10T18:58:46.986Z{"msg":"[0] client is_active_req TRUE, promote! session e980a471-a93a-4811-b0c4-ce07ed6fd25d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.977966629Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225022023-12-10T18:58:46.986Z{"msg":"[2] 6217da6e-3637-4b2d-a9dd-192703679ddd (e980a471-a93a-4811-b0c4-ce07ed6fd25d) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.97801094Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225032023-12-10T18:58:46.986Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.97804697Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225042023-12-10T18:58:46.986Z{"msg":"[2] client is_active_req TRUE, promote! session e980a471-a93a-4811-b0c4-ce07ed6fd25d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.978163424Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225052023-12-10T18:58:46.986Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.978437424Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225062023-12-10T18:58:46.986Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.978558186Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225072023-12-10T18:58:46.986Z{"msg":"6217da6e-3637-4b2d-a9dd-192703679ddd WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.978635293Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225082023-12-10T18:58:46.986Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.978755735Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225092023-12-10T18:58:46.986Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.97885292Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225102023-12-10T18:58:46.987Z{"msg":"6217da6e-3637-4b2d-a9dd-192703679ddd WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.978893905Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225112023-12-10T18:58:46.987Z{"msg":"[1] 6217da6e-3637-4b2d-a9dd-192703679ddd (e980a471-a93a-4811-b0c4-ce07ed6fd25d) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.978964519Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225122023-12-10T18:58:46.994Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.979006132Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225132023-12-10T18:58:46.995Z{"msg":"[1] client is_active_req TRUE, promote! session e980a471-a93a-4811-b0c4-ce07ed6fd25d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.979048753Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225142023-12-10T18:58:46.995Z{"msg":"[0] 6217da6e-3637-4b2d-a9dd-192703679ddd (e980a471-a93a-4811-b0c4-ce07ed6fd25d) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.979262996Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225152023-12-10T18:58:46.995Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.979356266Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225162023-12-10T18:58:46.995Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:46.979401838Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225172023-12-10T18:58:46.995Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.979475245Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225182023-12-10T18:58:46.995Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.979547404Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225192023-12-10T18:58:46.995Z{"msg":"6217da6e-3637-4b2d-a9dd-192703679ddd WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.979619525Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225202023-12-10T18:58:46.995Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.979683446Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225212023-12-10T18:58:46.995Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.979708599Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225222023-12-10T18:58:46.995Z{"msg":"[2] 6217da6e-3637-4b2d-a9dd-192703679ddd (e980a471-a93a-4811-b0c4-ce07ed6fd25d) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.979731664Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225232023-12-10T18:58:46.995Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.97977298Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225242023-12-10T18:58:46.995Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:46.97979563Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225252023-12-10T18:58:46.995Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.979815864Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225262023-12-10T18:58:46.995Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.979882619Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225272023-12-10T18:58:46.995Z{"msg":"6217da6e-3637-4b2d-a9dd-192703679ddd WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.979912997Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225282023-12-10T18:58:46.996Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.979957414Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225292023-12-10T18:58:46.996Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.979979845Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225302023-12-10T18:58:46.996Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.980001233Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225312023-12-10T18:58:46.996Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.980028225Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225322023-12-10T18:58:46.997Z{"msg":"6217da6e-3637-4b2d-a9dd-192703679ddd WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.980049764Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225332023-12-10T18:58:46.997Z{"msg":"[1] 6217da6e-3637-4b2d-a9dd-192703679ddd (e980a471-a93a-4811-b0c4-ce07ed6fd25d) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.980538717Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225342023-12-10T18:58:46.997Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.980645515Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225352023-12-10T18:58:46.997Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:46.980690582Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225362023-12-10T18:58:46.997Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.980760478Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225372023-12-10T18:58:46.997Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.980828988Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225382023-12-10T18:58:46.997Z{"msg":"6217da6e-3637-4b2d-a9dd-192703679ddd WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.980901351Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225392023-12-10T18:58:46.997Z{"msg":"[0]R flush_numbers[0..12]: [321, 321, 319, 320, 320, 321, 321, 315, 313, 309, 321, 318]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:184
225402023-12-10T18:58:46.997ZWait for a query_work_queue command to finish before sending IO
225412023-12-10T18:58:46.997Z,"time":"2023-12-10T18:58:46.980944645Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225422023-12-10T18:58:46.997Z{"msg":"[0]R generation[0..12]: [183, 183, 181, 182, 182, 183, 183, 179, 178, 176, 183, 180]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.981016507Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225432023-12-10T18:58:46.997Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.981063675Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225442023-12-10T18:58:46.997Z{"msg":"[1]R flush_numbers[0..12]: [321, 321, 319, 320, 320, 321, 321, 315, 313, 309, 321, 318]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.981114874Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225452023-12-10T18:58:46.997Z{"msg":"[1]R generation[0..12]: [183, 183, 181, 182, 182, 183, 183, 179, 178, 176, 183, 180]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.981220432Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225462023-12-10T18:58:46.997Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.981266326Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225472023-12-10T18:58:46.998Z{"msg":"[2]R flush_numbers[0..12]: [321, 321, 319, 320, 320, 321, 321, 315, 313, 309, 321, 318]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.981305843Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225482023-12-10T18:58:46.998Z{"msg":"[2]R generation[0..12]: [183, 183, 181, 182, 182, 183, 183, 179, 178, 176, 183, 180]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.981373683Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225492023-12-10T18:58:46.998Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.981419474Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225502023-12-10T18:58:46.998Z{"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.981459505Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225512023-12-10T18:58:46.998Z{"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.981499746Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225522023-12-10T18:58:46.998Z{"msg":"Next flush: 322","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.9816031Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225532023-12-10T18:58:46.998Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.981665313Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225542023-12-10T18:58:46.998Z{"msg":"No downstairs repair required","v":0,"name"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
225552023-12-10T18:58:46.998Z:"crucible","level":30,"time":"2023-12-10T18:58:46.981757648Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225562023-12-10T18:58:46.998Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.981807806Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225572023-12-10T18:58:46.998Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.981831395Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225582023-12-10T18:58:46.998Z{"msg":"6217da6e-3637-4b2d-a9dd-192703679ddd is now active with session: e980a471-a93a-4811-b0c4-ce07ed6fd25d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.981874371Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225592023-12-10T18:58:46.998Z{"msg":"6217da6e-3637-4b2d-a9dd-192703679ddd Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.981898424Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225602023-12-10T18:58:46.998Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.981919312Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225612023-12-10T18:58:46.998Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.98195976Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225622023-12-10T18:58:46.998Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.981982161Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225632023-12-10T18:58:46.998Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.982022843Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225642023-12-10T18:58:46.999Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.982044757Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225652023-12-10T18:58:46.999Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.982084932Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225662023-12-10T18:58:46.999Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.982106972Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225672023-12-10T18:58:46.999Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.982146895Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225682023-12-10T18:58:46.999Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.982168763Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225692023-12-10T18:58:46.999Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.982209026Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225702023-12-10T18:58:46.999Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:46.982230647Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1898,"session_id":"e980a471-a93a-4811-b0c4-ce07ed6fd25d"}
225712023-12-10T18:58:46.999ZLoading write count information from file "/tmp/test_repair_verify.data"
225722023-12-10T18:58:46.999ZRun Repair workload
225732023-12-10T18:58:46.999Z01/30 Write block 225 len 36864 data: 14 18 17 16 16 21 20 19 16
225742023-12-10T18:58:46.999Z02/30 Flush
225752023-12-10T18:58:46.999Z03/30 Write block 336 len 24576 data: 13 13 11 15 14 14
225762023-12-10T18:58:46.999Z04/30 Read block 510 len 8192
225772023-12-10T18:58:46.999Z05/30 Read block 35 len 24576
225782023-12-10T18:58:47.003Z06/30 Write block 79 len 28672 data: 17 15 18 19 21 21 20
225792023-12-10T18:58:47.003Z07/30 Write block 571 len 24576 data: 16 14 14 16 17 15
225802023-12-10T18:58:47.003Z08/30 Write block 236 len 32768 data: 17 16 15 19 20 21 23 20
225812023-12-10T18:58:47.003Z09/30 Flush
225822023-12-10T18:58:47.003Z10/30 Write block 43 len 16384 data: 9 9 12 10
225832023-12-10T18:58:47.003Z11/30 Read block 69 len 4096
225842023-12-10T18:58:47.006Z12/30 Write block 198 len 40960 data: 17 16 15 18 17 18 15 14 10 11
225852023-12-10T18:58:47.006Z13/30 Read block 88 len 4096
225862023-12-10T18:58:47.009Z14/30 Read block 119 len 40960
225872023-12-10T18:58:47.012Z15/30 Read block 436 len 40960
225882023-12-10T18:58:47.015Z16/30 Read block 93 len 36864
225892023-12-10T18:58:47.019Z17/30 Read block 530 len 28672
225902023-12-10T18:58:47.023Z18/30 Write block 466 len 8192 data: 12 14
225912023-12-10T18:58:47.023Z19/30 Write block 24 len 8192 data: 16 15
225922023-12-10T18:58:47.023Z20/30 Write block 126 len 32768 data: 18 16 17 15 13 12 10 10
225932023-12-10T18:58:47.026Z21/30 Write block 574 len 32768 data: 17 18 16 15 12 11 9 9
225942023-12-10T18:58:47.026Z22/30 Read block 458 len 20480
225952023-12-10T18:58:47.026Z23/30 Read block 22 len 32768
225962023-12-10T18:58:47.029Z24/30 Read block 165 len 28672
225972023-12-10T18:58:47.032Z25/30 Write block 88 len 8192 data: 21 19
225982023-12-10T18:58:47.032Z26/30 Read block 192 len 20480
225992023-12-10T18:58:47.035Z27/30 Read block 242 len 24576
226002023-12-10T18:58:47.041Z28/30 Write block 580 len 12288 data: 10 10 10
226012023-12-10T18:58:47.041Z29/30 Read block 154 len 36864
226022023-12-10T18:58:47.044Z30/30 Read block 569 len 4096
226032023-12-10T18:58:47.044Z----------------------------------------------------------------
226042023-12-10T18:58:47.044Z Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:21
226052023-12-10T18:58:47.044ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
226062023-12-10T18:58:47.045Z 10 Acked 1009 Write 4 Done Sent Done false
226072023-12-10T18:58:47.045Z 11 Acked 1010 Read 1 Done Sent Done false
226082023-12-10T18:58:47.045Z 12 Acked 1011 Write 10 Done Sent Done false
226092023-12-10T18:58:47.045Z 13 Acked 1012 Read 1 Done Sent Done false
226102023-12-10T18:58:47.045Z 14 Acked 1013 Read 10 Done Sent Done false
226112023-12-10T18:58:47.045Z 15 Acked 1014 Read 10 Done Sent Done false
226122023-12-10T18:58:47.045Z 16 Acked 1015 Read 9 Done Sent Done false
226132023-12-10T18:58:47.045Z 17 Acked 1016 Read 7 Done Sent Done false
226142023-12-10T18:58:47.045Z 18 Acked 1017 Write 2 Done Sent Done false
226152023-12-10T18:58:47.045Z 19 Acked 1018 Write 2 Done Sent Done false
226162023-12-10T18:58:47.046Z 20 Acked 1019 Write 8 Done Sent Done false
226172023-12-10T18:58:47.046Z 21 Acked 1020 Write 8 Done Sent Done false
226182023-12-10T18:58:47.046Z 22 Acked 1021 Read 5 Done Sent Done false
226192023-12-10T18:58:47.046Z 23 Acked 1022 Read 8 Done Sent Done false
226202023-12-10T18:58:47.046Z 24 Acked 1023 Read 7 Done Sent Done false
226212023-12-10T18:58:47.046Z 25 Acked 1024 Write 2 Done Sent Done false
226222023-12-10T18:58:47.046Z 26 Acked 1025 Read 5 Done Sent Done false
226232023-12-10T18:58:47.046Z 27 Acked 1026 Read 6 Done Sent Done false
226242023-12-10T18:58:47.046Z 28 Acked 1027 Write 3 Done Sent Done false
226252023-12-10T18:58:47.046Z 29 Acked 1028 Read 9 Done Sent Done false
226262023-12-10T18:58:47.046Z 30 Acked 1029 Read 1 Done Sent Done false
226272023-12-10T18:58:47.046Z STATES DS:0 DS:1 DS:2 TOTAL
226282023-12-10T18:58:47.046Z New 0 0 0 0
226292023-12-10T18:58:47.047Z Sent 0 21 0 21
226302023-12-10T18:58:47.047Z Done 21 0 21 42
226312023-12-10T18:58:47.047Z Skipped 0 0 0 0
226322023-12-10T18:58:47.047Z Error 0 0 0 0
226332023-12-10T18:58:47.047ZLast Flush: 1008 1008 1008
226342023-12-10T18:58:47.048ZDownstairs last five completed: 1008 1007 1006 1005 1004 1003
226352023-12-10T18:58:47.048ZUpstairs last five completed: 30 29 28 27 26 25
226362023-12-10T18:58:47.048ZWrote out file "/tmp/test_repair_verify.data"
226372023-12-10T18:58:47.051Z
226382023-12-10T18:58:47.053ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
226392023-12-10T18:58:47.080ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
226402023-12-10T18:58:47.080Z 0 000-019 183 183 183 321 321 321 F F F
226412023-12-10T18:58:47.083Z 1 020-039 183 183 183 321 321 321 T F T <---
226422023-12-10T18:58:47.083Z 2 040-059 181 181 181 319 319 319 T F T <---
226432023-12-10T18:58:47.083Z 3 060-079 184 184 184 323 323 323 F F F
226442023-12-10T18:58:47.083Z 4 080-099 184 184 184 323 323 323 T F T <---
226452023-12-10T18:58:47.083Z 5 100-119 183 183 183 321 321 321 F F F
226462023-12-10T18:58:47.083Z 6 120-139 183 183 183 321 321 321 T F T <---
226472023-12-10T18:58:47.083Z 7 140-159 179 179 179 315 315 315 F F F
226482023-12-10T18:58:47.083Z 8 160-179 178 178 178 313 313 313 F F F
226492023-12-10T18:58:47.084Z 9 180-199 176 176 176 309 309 309 T F T <---
226502023-12-10T18:58:47.084Z 10 200-219 183 183 183 321 321 321 T F T <---
226512023-12-10T18:58:47.084Z 11 220-239 184 184 184 323 323 323 F F F
226522023-12-10T18:58:47.084Z 12 240-259 184 184 184 323 323 323 F F F
226532023-12-10T18:58:47.084Z 13 260-279 183 183 183 321 321 321 F F F
226542023-12-10T18:58:47.084Z 14 280-299 183 183 183 321 321 321 F F F
226552023-12-10T18:58:47.084Z 15 300-319 182 182 182 320 320 320 F F F
226562023-12-10T18:58:47.084Z 16 320-339 184 184 184 323 323 323 F F F
226572023-12-10T18:58:47.084Z 17 340-359 184 184 184 323 323 323 F F F
226582023-12-10T18:58:47.084Z 18 360-379 182 182 182 320 320 320 F F F
226592023-12-10T18:58:47.084Z 19 380-399 183 183 183 321 321 321 F F F
226602023-12-10T18:58:47.084Z 20 400-419 181 181 181 319 319 319 F F F
226612023-12-10T18:58:47.084Z 21 420-439 182 182 182 320 320 320 F F F
226622023-12-10T18:58:47.084Z 22 440-459 178 178 178 311 311 311 F F F
226632023-12-10T18:58:47.084Z 23 460-479 176 176 176 309 309 309 T F T <---
226642023-12-10T18:58:47.084Z 24 480-499 183 183 183 321 321 321 F F F
226652023-12-10T18:58:47.084Z 25 500-519 182 182 182 320 320 320 F F F
226662023-12-10T18:58:47.085Z 26 520-539 183 183 183 321 321 321 F F F
226672023-12-10T18:58:47.085Z 27 540-559 183 183 183 321 321 321 F F F
226682023-12-10T18:58:47.085Z 28 560-579 184 184 184 323 323 323 T F T <---
226692023-12-10T18:58:47.085Z 29 580-599 164 164 164 286 286 286 T F T <---
226702023-12-10T18:58:47.085ZMax gen: 184, Max flush: 323
226712023-12-10T18:58:47.085ZError: Difference in extent metadata found!
226722023-12-10T18:58:47.088ZA Difference in extent metadata is expected here
226732023-12-10T18:58:47.088ZOn loop 92
226742023-12-10T18:58:47.088Z
226752023-12-10T18:58:47.088ZVerifying data now
226762023-12-10T18:58:52.049ZNo change after verify
226772023-12-10T18:58:52.049ZLoop: 92 Downstairs dump after verify (and repair):
226782023-12-10T18:58:52.072ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
226792023-12-10T18:58:52.072Z 0 000-019 183 183 183 321 321 321 F F F
226802023-12-10T18:58:52.075Z 1 020-039 185 185 185 324 324 324 F F F
226812023-12-10T18:58:52.075Z 2 040-059 185 185 185 324 324 324 F F F
226822023-12-10T18:58:52.075Z 3 060-079 184 184 184 323 323 323 F F F
226832023-12-10T18:58:52.076Z 4 080-099 185 185 185 324 324 324 F F F
226842023-12-10T18:58:52.076Z 5 100-119 183 183 183 321 321 321 F F F
226852023-12-10T18:58:52.076Z 6 120-139 185 185 185 324 324 324 F F F
226862023-12-10T18:58:52.076Z 7 140-159 179 179 179 315 315 315 F F F
226872023-12-10T18:58:52.076Z 8 160-179 178 178 178 313 313 313 F F F
226882023-12-10T18:58:52.077Z 9 180-199 185 185 185 324 324 324 F F F
226892023-12-10T18:58:52.077Z 10 200-219 185 185 185 324 324 324 F F F
226902023-12-10T18:58:52.077Z 11 220-239 184 184 184 323 323 323 F F F
226912023-12-10T18:58:52.077Z 12 240-259 184 184 184 323 323 323 F F F
226922023-12-10T18:58:52.077Z 13 260-279 183 183 183 321 321 321 F F F
226932023-12-10T18:58:52.077Z 14 280-299 183 183 183 321 321 321 F F F
226942023-12-10T18:58:52.077Z 15 300-319 182 182 182 320 320 320 F F F
226952023-12-10T18:58:52.077Z 16 320-339 184 184 184 323 323 323 F F F
226962023-12-10T18:58:52.077Z 17 340-359 184 184 184 323 323 323 F F F
226972023-12-10T18:58:52.077Z 18 360-379 182 182 182 320 320 320 F F F
226982023-12-10T18:58:52.077Z 19 380-399 183 183 183 321 321 321 F F F
226992023-12-10T18:58:52.077Z 20 400-419 181 181 181 319 319 319 F F F
227002023-12-10T18:58:52.077Z 21 420-439 182 182 182 320 320 320 F F F
227012023-12-10T18:58:52.077Z 22 440-459 178 178 178 311 311 311 F F F
227022023-12-10T18:58:52.077Z 23 460-479 185 185 185 324 324 324 F F F
227032023-12-10T18:58:52.078Z 24 480-499 183 183 183 321 321 321 F F F
227042023-12-10T18:58:52.078Z 25 500-519 182 182 182 320 320 320 F F F
227052023-12-10T18:58:52.078Z 26 520-539 183 183 183 321 321 321 F F F
227062023-12-10T18:58:52.078Z 27 540-559 183 183 183 321 321 321 F F F
227072023-12-10T18:58:52.078Z 28 560-579 185 185 185 324 324 324 F F F
227082023-12-10T18:58:52.078Z 29 580-599 185 185 185 324 324 324 F F F
227092023-12-10T18:58:52.078ZMax gen: 185, Max flush: 324
227102023-12-10T18:58:52.081Z
227112023-12-10T18:58:52.081ZBegin loop 93
227122023-12-10T18:58:52.081ZDownstairs to restart: 2
227132023-12-10T18:58:52.108Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.106249066Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906}
227142023-12-10T18:58:52.108Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.106669644Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906}
227152023-12-10T18:58:52.108Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.106725396Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906}
227162023-12-10T18:58:52.108Z{"msg":"Crucible a2d72067-2a4c-4ec1-9d54-633e97e828dd has session id: 15b63cda-5b42-4245-97ab-eff2f7f21927","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.10679516Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227172023-12-10T18:58:52.112Z{"msg":"Upstairs opts: Upstairs UUID: a2d72067-2a4c-4ec1-9d54-633e97e828dd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.106855726Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227182023-12-10T18:58:52.113Z{"msg":"Crucible stats registered with UUID: a2d72067-2a4c-4ec1-9d54-633e97e828dd","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.106899775Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227192023-12-10T18:58:52.113ZCrucible runtime is spawned
227202023-12-10T18:58:52.113Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.107116386Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"looper":"0","session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227212023-12-10T18:58:52.113Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30The guest has requested activation with gen:186
227222023-12-10T18:58:52.113Z,"time":"2023-12-10T18:58:52.10715747Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927","task":"up_listen"}
227232023-12-10T18:58:52.113Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.107275809Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227242023-12-10T18:58:52.113Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.107376004Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227252023-12-10T18:58:52.114Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.107419768Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"looper":"2","session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227262023-12-10T18:58:52.114Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.107605163Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"looper":"1","session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227272023-12-10T18:58:52.114Z{"msg":"[0] a2d72067-2a4c-4ec1-9d54-633e97e828dd looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.107684164Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"looper":"0","session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227282023-12-10T18:58:52.114Z{"msg":"[2] a2d72067-2a4c-4ec1-9d54-633e97e828dd looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.107731372Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"looper":"2","session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227292023-12-10T18:58:52.114Z{"msg":"[1] a2d72067-2a4c-4ec1-9d54-633e97e828dd looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.107808285Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"looper":"1","session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227302023-12-10T18:58:52.114Z{"msg":"a2d72067-2a4c-4ec1-9d54-633e97e828dd active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.108204603Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227312023-12-10T18:58:52.114Z{"msg":"Set desired generation to :186","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.108278765Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227322023-12-10T18:58:52.114Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.108439542Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227332023-12-10T18:58:52.114Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.108941476Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227342023-12-10T18:58:52.114Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.109044971Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227352023-12-10T18:58:52.114Z{"msg":"[0] a2d72067-2a4c-4ec1-9d54-633e97e828dd (15b63cda-5b42-4245-97ab-eff2f7f21927) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.109181145Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227362023-12-10T18:58:52.114Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.109243453Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227372023-12-10T18:58:52.114Z{"msg":"[0] client is_active_req TRUE, promote! session 15b63cda-5b42-4245-97ab-eff2f7f21927","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.10932758Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227382023-12-10T18:58:52.114Z{"msg":"[1] a2d72067-2a4c-4ec1-9d54-633e97e828dd (15b63cda-5b42-4245-97ab-eff2f7f21927) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.10943637Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227392023-12-10T18:58:52.114Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.109466383Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227402023-12-10T18:58:52.114Z{"msg":"[1] client is_active_req TRUE, promote! session 15b63cda-5b42-4245-97ab-eff2f7f21927","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.109525893Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227412023-12-10T18:58:52.114Z{"msg":"[2] a2d72067-2a4c-4ec1-9d54-633e97e828dd (15b63cda-5b42-4245-97ab-eff2f7f21927) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.109629583Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227422023-12-10T18:58:52.115Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.109725086Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227432023-12-10T18:58:52.115Z{"msg":"[2] client is_active_req TRUE, promote! session 15b63cda-5b42-4245-97ab-eff2f7f21927","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.109819978Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227442023-12-10T18:58:52.121Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.110213594Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227452023-12-10T18:58:52.121Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.110282248Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227462023-12-10T18:58:52.122Z{"msg":"a2d72067-2a4c-4ec1-9d54-633e97e828dd WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.110308251Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227472023-12-10T18:58:52.122Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.110584262Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227482023-12-10T18:58:52.122Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.110711722Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227492023-12-10T18:58:52.122Z{"msg":"a2d72067-2a4c-4ec1-9d54-633e97e828dd WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.11076134Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227502023-12-10T18:58:52.122Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.110993674Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227512023-12-10T18:58:52.122Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.11107532Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227522023-12-10T18:58:52.122Z{"msg":"a2d72067-2a4c-4ec1-9d54-633e97e828dd WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.111147792Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227532023-12-10T18:58:52.122Z{"msg":"[1] a2d72067-2a4c-4ec1-9d54-633e97e828dd (15b63cda-5b42-4245-97ab-eff2f7f21927) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.111192758Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227542023-12-10T18:58:52.122Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.111261665Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227552023-12-10T18:58:52.122Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:52.111301764Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227562023-12-10T18:58:52.122Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.111341432Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227572023-12-10T18:58:52.122Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.111379497Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227582023-12-10T18:58:52.122Z{"msg":"a2d72067-2a4c-4ec1-9d54-633e97e828dd WaitActive WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.1114482Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227592023-12-10T18:58:52.122Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.111486814Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227602023-12-10T18:58:52.122Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.111595414Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227612023-12-10T18:58:52.122Z{"msg":"[0] a2d72067-2a4c-4ec1-9d54-633e97e828dd (15b63cda-5b42-4245-97ab-eff2f7f21927) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.111642057Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227622023-12-10T18:58:52.123Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.111719994Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227632023-12-10T18:58:52.123Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:52.111824362Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227642023-12-10T18:58:52.123Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.111873338Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227652023-12-10T18:58:52.123Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.111896684Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227662023-12-10T18:58:52.123Z{"msg":"a2d72067-2a4c-4ec1-9d54-633e97e828dd WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.111917534Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227672023-12-10T18:58:52.123Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:186
227682023-12-10T18:58:52.123Z,"time":"2023-12-10T18:58:52.111989166Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"Wait for a query_work_queue command to finish before sending IO
227692023-12-10T18:58:52.123Z15b63cda-5b42-4245-97ab-eff2f7f21927"}
227702023-12-10T18:58:52.123Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.112018856Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227712023-12-10T18:58:52.123Z{"msg":"[2] a2d72067-2a4c-4ec1-9d54-633e97e828dd (15b63cda-5b42-4245-97ab-eff2f7f21927) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.112146502Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227722023-12-10T18:58:52.123Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.112257111Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227732023-12-10T18:58:52.123Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:52.112299939Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227742023-12-10T18:58:52.123Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.112338159Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227752023-12-10T18:58:52.123Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.11237625Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227762023-12-10T18:58:52.124Z{"msg":"a2d72067-2a4c-4ec1-9d54-633e97e828dd WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.112447641Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227772023-12-10T18:58:52.124Z{"msg":"[0]R flush_numbers[0..12]: [321, 324, 324, 323, 324, 321, 324, 315, 313, 324, 324, 323]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.112523656Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227782023-12-10T18:58:52.124Z{"msg":"[0]R generation[0..12]: [183, 185, 185, 184, 185, 183, 185, 179, 178, 185, 185, 184]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.112600618Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227792023-12-10T18:58:52.125Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.112694591Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227802023-12-10T18:58:52.125Z{"msg":"[1]R flush_numbers[0..12]: [321, 324, 324, 323, 324, 321, 324, 315, 313, 324, 324, 323]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.112740171Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227812023-12-10T18:58:52.125Z{"msg":"[1]R generation[0..12]: [183, 185, 185, 184, 185, 183, 185, 179, 178, 185, 185, 184]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.112811914Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227822023-12-10T18:58:52.125ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
227832023-12-10T18:58:52.125Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.112891104Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227842023-12-10T18:58:52.125Z{"msg":"[2]R flush_numbers[0..12]: [321, 324, 324, 323, 324, 321, 324, 315, 313, 324, 324, 323]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.112947608Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227852023-12-10T18:58:52.125Z{"msg":"[2]R generation[0..12]: [183, 185, 185, 184, 185, 183, 185, 179, 178, 185, 185, 184]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.112972404Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227862023-12-10T18:58:52.125Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.112994192Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227872023-12-10T18:58:52.125Z{"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.113013524Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227882023-12-10T18:58:52.125Z{"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.113053601Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227892023-12-10T18:58:52.125Z{"msg":"Next flush: 325","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.113075355Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227902023-12-10T18:58:52.125Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.113095238Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227912023-12-10T18:58:52.125Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.11311467Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227922023-12-10T18:58:52.125Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.113154649Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227932023-12-10T18:58:52.125Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.113176884Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227942023-12-10T18:58:52.125Z{"msg":"a2d72067-2a4c-4ec1-9d54-633e97e828dd is now active with session: 15b63cda-5b42-4245-97ab-eff2f7f21927","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.113217884Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227952023-12-10T18:58:52.125Z{"msg":"a2d72067-2a4c-4ec1-9d54-633e97e828dd Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.113240475Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227962023-12-10T18:58:52.126Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.113280827Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227972023-12-10T18:58:52.126Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.113302427Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227982023-12-10T18:58:52.126Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.113342008Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
227992023-12-10T18:58:52.126Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.113363607Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
228002023-12-10T18:58:52.126Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"Loading write count information from file "/tmp/test_repair_verify.data"
228012023-12-10T18:58:52.126Z2023-12-10T18:58:52.113402478Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
228022023-12-10T18:58:52.126Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30Run Repair workload
228032023-12-10T18:58:52.126Z,"time":"2023-12-10T18:58:52.113431158Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
228042023-12-10T18:58:52.126Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.113536709Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
228052023-12-10T18:58:52.126Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.113562768Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
228062023-12-10T18:58:52.126Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.1136096Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
228072023-12-10T18:58:52.126Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.113634328Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
228082023-12-10T18:58:52.126Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:52.113675783Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1906,"session_id":"15b63cda-5b42-4245-97ab-eff2f7f21927"}
228092023-12-10T18:58:52.127Z01/30 Write block 442 len 28672 data: 8 10 10 12 13 13 15
228102023-12-10T18:58:52.127Z02/30 Flush
228112023-12-10T18:58:52.131Z03/30 Write block 174 len 40960 data: 13 10 10 12 10 9 9 11 14 14
228122023-12-10T18:58:52.131Z04/30 Read block 46 len 8192
228132023-12-10T18:58:52.131Z05/30 Read block 571 len 8192
228142023-12-10T18:58:52.132Z06/30 Read block 577 len 20480
228152023-12-10T18:58:52.132Z07/30 Read block 94 len 8192
228162023-12-10T18:58:52.132Z08/30 Write block 403 len 32768 data: 18 15 18 15 15 18 17 15
228172023-12-10T18:58:52.132Z09/30 Write block 551 len 16384 data: 9 9 9 9
228182023-12-10T18:58:52.132Z10/30 Flush
228192023-12-10T18:58:52.135Z11/30 Write block 175 len 24576 data: 11 11 13 11 10 10
228202023-12-10T18:58:52.135Z12/30 Read block 144 len 12288
228212023-12-10T18:58:52.139Z13/30 Write block 47 len 32768 data: 10 10 12 12 11 10 10 12
228222023-12-10T18:58:52.139Z14/30 Read block 408 len 16384
228232023-12-10T18:58:52.142Z15/30 Read block 127 len 32768
228242023-12-10T18:58:52.145Z16/30 Write block 325 len 16384 data: 14 14 15 14
228252023-12-10T18:58:52.145Z17/30 Write block 379 len 28672 data: 20 21 16 15 12 14 16
228262023-12-10T18:58:52.145Z18/30 Read block 503 len 4096
228272023-12-10T18:58:52.148Z19/30 Write block 498 len 36864 data: 14 11 14 16 16 17 12 13 13
228282023-12-10T18:58:52.151Z20/30 Write block 423 len 16384 data: 16 18 15 16
228292023-12-10T18:58:52.151Z21/30 Write block 431 len 8192 data: 13 12
228302023-12-10T18:58:52.151Z22/30 Read block 65 len 20480
228312023-12-10T18:58:52.154Z23/30 Read block 321 len 28672
228322023-12-10T18:58:52.159Z24/30 Write block 133 len 4096 data: 11
228332023-12-10T18:58:52.159Z25/30 Read block 539 len 40960
228342023-12-10T18:58:52.162Z26/30 Read block 557 len 20480
228352023-12-10T18:58:52.162Z27/30 Read block 236 len 12288
228362023-12-10T18:58:52.163Z28/30 Read block 203 len 20480
228372023-12-10T18:58:52.165Z29/30 Read block 586 len 36864
228382023-12-10T18:58:52.169Z30/30 Write block 577 len 36864 data: 16 13 12 11 11 11 9 10 8
228392023-12-10T18:58:52.169Z----------------------------------------------------------------
228402023-12-10T18:58:52.169Z Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:30
228412023-12-10T18:58:52.169ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
228422023-12-10T18:58:52.169Z 1 Acked 1000 Write 7 Done Done Sent false
228432023-12-10T18:58:52.169Z 2 Acked 1001 Flush 0 Done Done Sent false
228442023-12-10T18:58:52.169Z 3 Acked 1002 Write 10 Done Done Sent false
228452023-12-10T18:58:52.169Z 4 Acked 1003 Read 2 Done Done Sent false
228462023-12-10T18:58:52.169Z 5 Acked 1004 Read 2 Done Done Sent false
228472023-12-10T18:58:52.170Z 6 Acked 1005 Read 5 Done Done Sent false
228482023-12-10T18:58:52.170Z 7 Acked 1006 Read 2 Done Done Sent false
228492023-12-10T18:58:52.170Z 8 Acked 1007 Write 8 Done Done Sent false
228502023-12-10T18:58:52.170Z 9 Acked 1008 Write 4 Done Done Sent false
228512023-12-10T18:58:52.170Z 10 Acked 1009 Flush 0 Done Done Sent false
228522023-12-10T18:58:52.170Z 11 Acked 1010 Write 6 Done Done Sent false
228532023-12-10T18:58:52.170Z 12 Acked 1011 Read 3 Done Done Sent false
228542023-12-10T18:58:52.170Z 13 Acked 1012 Write 8 Done Done Sent false
228552023-12-10T18:58:52.170Z 14 Acked 1013 Read 4 Done Done Sent false
228562023-12-10T18:58:52.170Z 15 Acked 1014 Read 8 Done Done Sent false
228572023-12-10T18:58:52.170Z 16 Acked 1015 Write 4 Done Done Sent false
228582023-12-10T18:58:52.170Z 17 Acked 1016 Write 7 Done Done Sent false
228592023-12-10T18:58:52.170Z 18 Acked 1017 Read 1 Done Done Sent false
228602023-12-10T18:58:52.170Z 19 Acked 1018 Write 9 Done Done Sent false
228612023-12-10T18:58:52.170Z 20 Acked 1019 Write 4 Done Done Sent false
228622023-12-10T18:58:52.170Z 21 Acked 1020 Write 2 Done Done Sent false
228632023-12-10T18:58:52.170Z 22 Acked 1021 Read 5 Done Done Sent false
228642023-12-10T18:58:52.171Z 23 Acked 1022 Read 7 Done Done Sent false
228652023-12-10T18:58:52.171Z 24 Acked 1023 Write 1 Done Done Sent false
228662023-12-10T18:58:52.171Z 25 Acked 1024 Read 10 Done Done Sent false
228672023-12-10T18:58:52.171Z 26 Acked 1025 Read 5 Done Done Sent false
228682023-12-10T18:58:52.171Z 27 Acked 1026 Read 3 Done Done Sent false
228692023-12-10T18:58:52.171Z 28 Acked 1027 Read 5 Done Done Sent false
228702023-12-10T18:58:52.171Z 29 Acked 1028 Read 9 Done Done Sent false
228712023-12-10T18:58:52.171Z 30 Acked 1029 Write 9 Sent Sent Sent false
228722023-12-10T18:58:52.171Z STATES DS:0 DS:1 DS:2 TOTAL
228732023-12-10T18:58:52.171Z New 0 0 0 0
228742023-12-10T18:58:52.171Z Sent 1 1 30 32
228752023-12-10T18:58:52.171Z Done 29 29 0 58
228762023-12-10T18:58:52.171Z Skipped 0 0 0 0
228772023-12-10T18:58:52.171Z Error 0 0 0 0
228782023-12-10T18:58:52.171ZLast Flush: 1009 1009 0
228792023-12-10T18:58:52.171ZDownstairs last five completed:
228802023-12-10T18:58:52.171ZUpstairs last five completed: 30 29 28 27 26 25
228812023-12-10T18:58:52.172ZWrote out file "/tmp/test_repair_verify.data"
228822023-12-10T18:58:52.175Z
228832023-12-10T18:58:52.179ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
228842023-12-10T18:58:52.208ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
228852023-12-10T18:58:52.208Z 0 000-019 183 183 183 321 321 321 F F F
228862023-12-10T18:58:52.211Z 1 020-039 185 185 185 324 324 324 F F F
228872023-12-10T18:58:52.211Z 2 040-059 185 185 185 324 324 324 T T F <---
228882023-12-10T18:58:52.211Z 3 060-079 184 184 184 323 323 323 F F F
228892023-12-10T18:58:52.211Z 4 080-099 185 185 185 324 324 324 F F F
228902023-12-10T18:58:52.211Z 5 100-119 183 183 183 321 321 321 F F F
228912023-12-10T18:58:52.212Z 6 120-139 185 185 185 324 324 324 T T F <---
228922023-12-10T18:58:52.212Z 7 140-159 179 179 179 315 315 315 F F F
228932023-12-10T18:58:52.212Z 8 160-179 186 186 178 326 326 313 T T F <---
228942023-12-10T18:58:52.212Z 9 180-199 186 186 185 326 326 324 T T F <---
228952023-12-10T18:58:52.212Z 10 200-219 185 185 185 324 324 324 F F F
228962023-12-10T18:58:52.212Z 11 220-239 184 184 184 323 323 323 F F F
228972023-12-10T18:58:52.212Z 12 240-259 184 184 184 323 323 323 F F F
228982023-12-10T18:58:52.212Z 13 260-279 183 183 183 321 321 321 F F F
228992023-12-10T18:58:52.212Z 14 280-299 183 183 183 321 321 321 F F F
229002023-12-10T18:58:52.212Z 15 300-319 182 182 182 320 320 320 F F F
229012023-12-10T18:58:52.212Z 16 320-339 184 184 184 323 323 323 T T F <---
229022023-12-10T18:58:52.212Z 17 340-359 184 184 184 323 323 323 F F F
229032023-12-10T18:58:52.212Z 18 360-379 182 182 182 320 320 320 T T F <---
229042023-12-10T18:58:52.212Z 19 380-399 183 183 183 321 321 321 T T F <---
229052023-12-10T18:58:52.212Z 20 400-419 186 186 181 326 326 319 F F F <---
229062023-12-10T18:58:52.212Z 21 420-439 182 182 182 320 320 320 T T F <---
229072023-12-10T18:58:52.212Z 22 440-459 186 186 178 325 325 311 F F F <---
229082023-12-10T18:58:52.212Z 23 460-479 185 185 185 324 324 324 F F F
229092023-12-10T18:58:52.213Z 24 480-499 183 183 183 321 321 321 T T F <---
229102023-12-10T18:58:52.213Z 25 500-519 182 182 182 320 320 320 T T F <---
229112023-12-10T18:58:52.213Z 26 520-539 183 183 183 321 321 321 F F F
229122023-12-10T18:58:52.213Z 27 540-559 186 186 183 326 326 321 F F F <---
229132023-12-10T18:58:52.213Z 28 560-579 185 185 185 324 324 324 T T F <---
229142023-12-10T18:58:52.213Z 29 580-599 185 185 185 324 324 324 T T F <---
229152023-12-10T18:58:52.213ZMax gen: 186, Max flush: 326
229162023-12-10T18:58:52.213ZError: Difference in extent metadata found!
229172023-12-10T18:58:52.216ZA Difference in extent metadata is expected here
229182023-12-10T18:58:52.216ZOn loop 93
229192023-12-10T18:58:52.216Z
229202023-12-10T18:58:52.216ZVerifying data now
229212023-12-10T18:58:57.638ZNo change after verify
229222023-12-10T18:58:57.639ZLoop: 93 Downstairs dump after verify (and repair):
229232023-12-10T18:58:57.664ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
229242023-12-10T18:58:57.664Z 0 000-019 183 183 183 321 321 321 F F F
229252023-12-10T18:58:57.668Z 1 020-039 185 185 185 324 324 324 F F F
229262023-12-10T18:58:57.668Z 2 040-059 187 187 187 327 327 327 F F F
229272023-12-10T18:58:57.668Z 3 060-079 184 184 184 323 323 323 F F F
229282023-12-10T18:58:57.668Z 4 080-099 185 185 185 324 324 324 F F F
229292023-12-10T18:58:57.668Z 5 100-119 183 183 183 321 321 321 F F F
229302023-12-10T18:58:57.668Z 6 120-139 187 187 187 327 327 327 F F F
229312023-12-10T18:58:57.668Z 7 140-159 179 179 179 315 315 315 F F F
229322023-12-10T18:58:57.668Z 8 160-179 187 187 187 327 327 327 F F F
229332023-12-10T18:58:57.668Z 9 180-199 187 187 187 327 327 327 F F F
229342023-12-10T18:58:57.668Z 10 200-219 185 185 185 324 324 324 F F F
229352023-12-10T18:58:57.668Z 11 220-239 184 184 184 323 323 323 F F F
229362023-12-10T18:58:57.668Z 12 240-259 184 184 184 323 323 323 F F F
229372023-12-10T18:58:57.668Z 13 260-279 183 183 183 321 321 321 F F F
229382023-12-10T18:58:57.668Z 14 280-299 183 183 183 321 321 321 F F F
229392023-12-10T18:58:57.668Z 15 300-319 182 182 182 320 320 320 F F F
229402023-12-10T18:58:57.668Z 16 320-339 187 187 187 327 327 327 F F F
229412023-12-10T18:58:57.669Z 17 340-359 184 184 184 323 323 323 F F F
229422023-12-10T18:58:57.669Z 18 360-379 187 187 187 327 327 327 F F F
229432023-12-10T18:58:57.669Z 19 380-399 187 187 187 327 327 327 F F F
229442023-12-10T18:58:57.669Z 20 400-419 186 186 186 326 326 326 F F F
229452023-12-10T18:58:57.669Z 21 420-439 187 187 187 327 327 327 F F F
229462023-12-10T18:58:57.669Z 22 440-459 186 186 186 325 325 325 F F F
229472023-12-10T18:58:57.669Z 23 460-479 185 185 185 324 324 324 F F F
229482023-12-10T18:58:57.669Z 24 480-499 187 187 187 327 327 327 F F F
229492023-12-10T18:58:57.669Z 25 500-519 187 187 187 327 327 327 F F F
229502023-12-10T18:58:57.669Z 26 520-539 183 183 183 321 321 321 F F F
229512023-12-10T18:58:57.669Z 27 540-559 186 186 186 326 326 326 F F F
229522023-12-10T18:58:57.669Z 28 560-579 187 187 187 327 327 327 F F F
229532023-12-10T18:58:57.669Z 29 580-599 187 187 187 327 327 327 F F F
229542023-12-10T18:58:57.669ZMax gen: 187, Max flush: 327
229552023-12-10T18:58:57.669Z
229562023-12-10T18:58:57.669ZBegin loop 94
229572023-12-10T18:58:57.670ZDownstairs to restart: 2
229582023-12-10T18:58:57.695Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.693666863Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914}
229592023-12-10T18:58:57.695Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.694152875Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914}
229602023-12-10T18:58:57.700Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.694225648Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914}
229612023-12-10T18:58:57.700Z{"msg":"Crucible 1c447e47-0842-4417-8252-987e01a1a851 has session id: b7e2131b-ef18-4d65-8d3c-f6131ed08cda","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.694298832Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229622023-12-10T18:58:57.700Z{"msg":"Upstairs opts: Upstairs UUID: 1c447e47-0842-4417-8252-987e01a1a851, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.694341539Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229632023-12-10T18:58:57.702Z{"msg":"Crucible stats registered with UUID: 1c447e47-0842-4417-8252-987e01a1a851","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.694420963Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229642023-12-10T18:58:57.702ZCrucible runtime is spawned
229652023-12-10T18:58:57.702ZThe guest has requested activation with gen:188
229662023-12-10T18:58:57.702Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.694768948Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda","task":"up_listen"}
229672023-12-10T18:58:57.702Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.694827107Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"looper":"0","session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229682023-12-10T18:58:57.702Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.694907738Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229692023-12-10T18:58:57.703Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.694948798Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229702023-12-10T18:58:57.703Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.695061948Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"looper":"2","session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229712023-12-10T18:58:57.703Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.695191823Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"looper":"1","session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229722023-12-10T18:58:57.703Z{"msg":"[0] 1c447e47-0842-4417-8252-987e01a1a851 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.695299811Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"looper":"0","session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229732023-12-10T18:58:57.703Z{"msg":"[2] 1c447e47-0842-4417-8252-987e01a1a851 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.695404695Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"looper":"2","session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229742023-12-10T18:58:57.704Z{"msg":"[1] 1c447e47-0842-4417-8252-987e01a1a851 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.695486881Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"looper":"1","session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229752023-12-10T18:58:57.704Z{"msg":"1c447e47-0842-4417-8252-987e01a1a851 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.695957557Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229762023-12-10T18:58:57.704Z{"msg":"Set desired generation to :188","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.696023575Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229772023-12-10T18:58:57.704Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.696172982Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229782023-12-10T18:58:57.704Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.696588721Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229792023-12-10T18:58:57.704Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.696719643Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229802023-12-10T18:58:57.704Z{"msg":"[0] 1c447e47-0842-4417-8252-987e01a1a851 (b7e2131b-ef18-4d65-8d3c-f6131ed08cda) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.696948327Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229812023-12-10T18:58:57.704Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.697098552Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229822023-12-10T18:58:57.704Z{"msg":"[0] client is_active_req TRUE, promote! session b7e2131b-ef18-4d65-8d3c-f6131ed08cda","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.6971463Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229832023-12-10T18:58:57.704Z{"msg":"[1] 1c447e47-0842-4417-8252-987e01a1a851 (b7e2131b-ef18-4d65-8d3c-f6131ed08cda) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.69719058Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229842023-12-10T18:58:57.705Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.697230313Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229852023-12-10T18:58:57.705Z{"msg":"[1] client is_active_req TRUE, promote! session b7e2131b-ef18-4d65-8d3c-f6131ed08cda","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.697270876Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229862023-12-10T18:58:57.712Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.697739086Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229872023-12-10T18:58:57.712Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.697854305Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229882023-12-10T18:58:57.712Z{"msg":"1c447e47-0842-4417-8252-987e01a1a851 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.697898938Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229892023-12-10T18:58:57.712Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.698035171Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229902023-12-10T18:58:57.712Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.698090457Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229912023-12-10T18:58:57.712Z{"msg":"1c447e47-0842-4417-8252-987e01a1a851 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.698164366Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229922023-12-10T18:58:57.712Z{"msg":"[2] 1c447e47-0842-4417-8252-987e01a1a851 (b7e2131b-ef18-4d65-8d3c-f6131ed08cda) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.698211392Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229932023-12-10T18:58:57.713Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.69827986Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229942023-12-10T18:58:57.714Z{"msg":"[2] client is_active_req TRUE, promote! session b7e2131b-ef18-4d65-8d3c-f6131ed08cda","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.698321699Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229952023-12-10T18:58:57.714Z{"msg":"[0] 1c447e47-0842-4417-8252-987e01a1a851 (b7e2131b-ef18-4d65-8d3c-f6131ed08cda) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.698490003Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229962023-12-10T18:58:57.714Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.698632266Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229972023-12-10T18:58:57.714Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:57.698724301Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229982023-12-10T18:58:57.714Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.698766665Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
229992023-12-10T18:58:57.714Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.698803646Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230002023-12-10T18:58:57.714Z{"msg":"1c447e47-0842-4417-8252-987e01a1a851 WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.698828029Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230012023-12-10T18:58:57.714Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.698875161Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230022023-12-10T18:58:57.714Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.698898265Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230032023-12-10T18:58:57.714Z{"msg":"[1] 1c447e47-0842-4417-8252-987e01a1a851 (b7e2131b-ef18-4d65-8d3c-f6131ed08cda) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.698921214Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230042023-12-10T18:58:57.714Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.698941758Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230052023-12-10T18:58:57.714Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:57.698996766Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230062023-12-10T18:58:57.714Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.699038558Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230072023-12-10T18:58:57.714Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.699114155Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230082023-12-10T18:58:57.714Z{"msg":"1c447e47-0842-4417-8252-987e01a1a851 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.699180166Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230092023-12-10T18:58:57.714Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.69920679Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230102023-12-10T18:58:57.715Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.699264274Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230112023-12-10T18:58:57.715Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.699342129Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230122023-12-10T18:58:57.715Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.699426439Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230132023-12-10T18:58:57.715Z{"msg":"1c447e47-0842-4417-8252-987e01a1a851 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.699500687Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230142023-12-10T18:58:57.715Z{"msg":"[2] 1c447e47-0842-4417-8252-987e01a1a851 (b7e2131b-ef18-4d65-8d3c-f6131ed08cda) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.699947009Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230152023-12-10T18:58:57.715Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.700031641Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230162023-12-10T18:58:57.715Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:58:57.700104638Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230172023-12-10T18:58:57.715Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.700136088Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230182023-12-10T18:58:57.715Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.700158047Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230192023-12-10T18:58:57.715Z{"msg":"1c447e47-0842-4417-8252-987e01a1a851 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.700181215Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230202023-12-10T18:58:57.715Z{"msg":"[0]R flush_numbers[0..12]: [321, 324, 327, 323, 324, 321, 327, 315, 327, 327, 324, 323]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.700225127Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230212023-12-10T18:58:57.715Z{"msg":"[0]R generation[0..12]: [183, 185, 187, 184, 185, 183, 187, 179, 187, 187, 185, 184]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.700249077Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230222023-12-10T18:58:57.715Z{"msg":"The guest has finished waiting for activation with:188
230232023-12-10T18:58:57.716Z[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
230242023-12-10T18:58:57.716Z,"time":"2023-12-10T18:58:57.700299556Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230252023-12-10T18:58:57.716Z{"msg":"[1]R flush_numbers[0..12]: [321, 324, 327, 323, 324, 321, 327, 315, 327, 327, 324, 323]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.700325785Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230262023-12-10T18:58:57.716Z{"msg":"[1]R generation[0..12]: [183, 185, 187, 184, 185, 183, 187, 179, 187, 187, 185, 184]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.70036848Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230272023-12-10T18:58:57.716Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.700393217Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230282023-12-10T18:58:57.716Z{"msg":"[2]R flush_numbers[0..12]: [321, 324, 327, 323, 324, 321, 327, 315, 327, 327, 324, 323]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.700414845Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230292023-12-10T18:58:57.717Z{"msg":"[2]R generation[0..12]: [183, 185, 187, 184, 185, 183, 187, 179, 187, 187, 185, 184]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.700463288Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230302023-12-10T18:58:57.717Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.700487443Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230312023-12-10T18:58:57.717Z{"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.700507562Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230322023-12-10T18:58:57.717Z{"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.700528367Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230332023-12-10T18:58:57.717Z{"msg":"Next flush: 328","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.700559754Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230342023-12-10T18:58:57.718Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.700597971Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230352023-12-10T18:58:57.718Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.700670409Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230362023-12-10T18:58:57.718Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.700712052Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230372023-12-10T18:58:57.718Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.700785474Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230382023-12-10T18:58:57.718Z{"msg":"1c447e47-0842-4417-8252-987e01a1a851 is now active with session: b7e2131b-ef18-4d65-8d3c-f6131ed08cda","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.700831629Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230392023-12-10T18:58:57.718Z{"msg":"1c447e47-0842-4417-8252-987e01a1a851 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.70090074Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230402023-12-10T18:58:57.718Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.700971202Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230412023-12-10T18:58:57.718Z{"msg":"Set check for repair","v":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
230422023-12-10T18:58:57.718Z0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.701050353Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230432023-12-10T18:58:57.718Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.701114125Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230442023-12-10T18:58:57.718Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.701139472Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230452023-12-10T18:58:57.719Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.701180188Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230462023-12-10T18:58:57.719Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.701201723Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230472023-12-10T18:58:57.719Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.701241642Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230482023-12-10T18:58:57.719Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.701263731Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230492023-12-10T18:58:57.719Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.701303814Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230502023-12-10T18:58:57.719Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.701325629Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230512023-12-10T18:58:57.720Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:58:57.701365933Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1914,"session_id":"b7e2131b-ef18-4d65-8d3c-f6131ed08cda"}
230522023-12-10T18:58:57.720ZLoading write count information from file "/tmp/test_repair_verify.data"
230532023-12-10T18:58:57.720ZRun Repair workload
230542023-12-10T18:58:57.720Z01/30 Write block 377 len 24576 data: 20 23 21 22 17 16
230552023-12-10T18:58:57.720Z02/30 Flush
230562023-12-10T18:58:57.720Z03/30 Write block 540 len 36864 data: 19 22 22 21 19 15 14 14 14
230572023-12-10T18:58:57.724Z04/30 Write block 55 len 40960 data: 11 12 12 12 9 11 12 14 15 15
230582023-12-10T18:58:57.724Z05/30 Write block 370 len 16384 data: 11 15 16 17
230592023-12-10T18:58:57.724Z06/30 Write block 110 len 28672 data: 13 12 13 16 14 16 17
230602023-12-10T18:58:57.724Z07/30 Read block 335 len 16384
230612023-12-10T18:58:57.724Z08/30 Read block 253 len 20480
230622023-12-10T18:58:57.724Z09/30 Write block 288 len 4096 data: 15
230632023-12-10T18:58:57.725Z10/30 Write block 260 len 24576 data: 10 12 16 16 15 17
230642023-12-10T18:58:57.725Z11/30 Read block 525 len 28672
230652023-12-10T18:58:57.725Z12/30 Write block 345 len 36864 data: 15 14 18 17 19 16 16 20 17
230662023-12-10T18:58:57.725Z13/30 Read block 442 len 40960
230672023-12-10T18:58:57.728Z14/30 Read block 262 len 28672
230682023-12-10T18:58:57.731Z15/30 Read block 404 len 20480
230692023-12-10T18:58:57.731Z16/30 Write block 103 len 4096 data: 12
230702023-12-10T18:58:57.731Z17/30 Read block 199 len 12288
230712023-12-10T18:58:57.734Z18/30 Read block 117 len 28672
230722023-12-10T18:58:57.737Z19/30 Flush
230732023-12-10T18:58:57.740Z20/30 Flush
230742023-12-10T18:58:57.743Z21/30 Write block 330 len 36864 data: 14 12 16 15 13 12 14 14 12
230752023-12-10T18:58:57.743Z22/30 Flush
230762023-12-10T18:58:57.746Z23/30 Write block 136 len 28672 data: 5 3 6 8 7 6 7
230772023-12-10T18:58:57.746Z24/30 Write block 470 len 28672 data: 14 13 12 12 11 9 11
230782023-12-10T18:58:57.746Z25/30 Read block 52 len 16384
230792023-12-10T18:58:57.749Z26/30 Read block 210 len 40960
230802023-12-10T18:58:57.752Z27/30 Write block 137 len 36864 data: 4 7 9 8 7 8 6 8 8
230812023-12-10T18:58:57.755Z28/30 Read block 364 len 28672
230822023-12-10T18:58:57.761Z29/30 Write block 470 len 40960 data: 15 14 13 13 12 10 12 9 7 8
230832023-12-10T18:58:57.761Z30/30 Read block 355 len 32768
230842023-12-10T18:58:57.764Z----------------------------------------------------------------
230852023-12-10T18:58:57.765Z Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:28
230862023-12-10T18:58:57.765ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
230872023-12-10T18:58:57.766Z 3 Acked 1002 Write 9 Done Done Done false
230882023-12-10T18:58:57.766Z 4 Acked 1003 Write 10 Done Done Done false
230892023-12-10T18:58:57.766Z 5 Acked 1004 Write 4 Done Done Done false
230902023-12-10T18:58:57.766Z 6 Acked 1005 Write 7 Done Done Done false
230912023-12-10T18:58:57.766Z 7 Acked 1006 Read 4 Done Done Done false
230922023-12-10T18:58:57.766Z 8 Acked 1007 Read 5 Done Done Done false
230932023-12-10T18:58:57.766Z 9 Acked 1008 Write 1 Done Done Done false
230942023-12-10T18:58:57.766Z 10 Acked 1009 Write 6 Done Done Done false
230952023-12-10T18:58:57.766Z 11 Acked 1010 Read 7 Done Done Done false
230962023-12-10T18:58:57.766Z 12 Acked 1011 Write 9 Done Done Done false
230972023-12-10T18:58:57.766Z 13 Acked 1012 Read 10 Done Done Sent false
230982023-12-10T18:58:57.766Z 14 Acked 1013 Read 7 Done Done Sent false
230992023-12-10T18:58:57.766Z 15 Acked 1014 Read 5 Done Done Sent false
231002023-12-10T18:58:57.766Z 16 Acked 1015 Write 1 Done Done Sent false
231012023-12-10T18:58:57.766Z 17 Acked 1016 Read 3 Done Done Sent false
231022023-12-10T18:58:57.766Z 18 Acked 1017 Read 7 Done Done Sent false
231032023-12-10T18:58:57.766Z 19 Acked 1018 Flush 0 Done Done Sent false
231042023-12-10T18:58:57.767Z 20 Acked 1019 Flush 0 Done Done Sent false
231052023-12-10T18:58:57.767Z 21 Acked 1020 Write 9 Done Done Sent false
231062023-12-10T18:58:57.767Z 22 Acked 1021 Flush 0 Done Done Sent false
231072023-12-10T18:58:57.767Z 23 Acked 1022 Write 7 Done Done Sent false
231082023-12-10T18:58:57.767Z 24 Acked 1023 Write 7 Done Done Sent false
231092023-12-10T18:58:57.767Z 25 Acked 1024 Read 4 Done Done Sent false
231102023-12-10T18:58:57.767Z 26 Acked 1025 Read 10 Done Done Sent false
231112023-12-10T18:58:57.767Z 27 Acked 1026 Write 9 Done Done Sent false
231122023-12-10T18:58:57.767Z 28 Acked 1027 Read 7 Done Done Sent false
231132023-12-10T18:58:57.767Z 29 Acked 1028 Write 10 Done Done Sent false
231142023-12-10T18:58:57.767Z 30 Acked 1029 Read 8 Done Done Sent false
231152023-12-10T18:58:57.767Z STATES DS:0 DS:1 DS:2 TOTAL
231162023-12-10T18:58:57.767Z New 0 0 0 0
231172023-12-10T18:58:57.767Z Sent 0 0 18 18
231182023-12-10T18:58:57.767Z Done 28 28 10 66
231192023-12-10T18:58:57.767Z Skipped 0 0 0 0
231202023-12-10T18:58:57.767Z Error 0 0 0 0
231212023-12-10T18:58:57.767ZLast Flush: 1021 1021 1001
231222023-12-10T18:58:57.768ZDownstairs last five completed: 1001 1000
231232023-12-10T18:58:57.768ZUpstairs last five completed: 30 29 28 27 26 25
231242023-12-10T18:58:57.768ZWrote out file "/tmp/test_repair_verify.data"
231252023-12-10T18:58:57.771Z
231262023-12-10T18:58:57.774ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
231272023-12-10T18:58:57.799ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
231282023-12-10T18:58:57.799Z 0 000-019 183 183 183 321 321 321 F F F
231292023-12-10T18:58:57.802Z 1 020-039 185 185 185 324 324 324 F F F
231302023-12-10T18:58:57.802Z 2 040-059 188 188 187 329 329 327 F F T <---
231312023-12-10T18:58:57.802Z 3 060-079 188 188 184 329 329 323 F F T <---
231322023-12-10T18:58:57.802Z 4 080-099 185 185 185 324 324 324 F F F
231332023-12-10T18:58:57.802Z 5 100-119 188 188 183 329 329 321 F F T <---
231342023-12-10T18:58:57.802Z 6 120-139 187 187 187 327 327 327 T T F <---
231352023-12-10T18:58:57.802Z 7 140-159 179 179 179 315 315 315 T T F <---
231362023-12-10T18:58:57.802Z 8 160-179 187 187 187 327 327 327 F F F
231372023-12-10T18:58:57.803Z 9 180-199 187 187 187 327 327 327 F F F
231382023-12-10T18:58:57.803Z 10 200-219 185 185 185 324 324 324 F F F
231392023-12-10T18:58:57.803Z 11 220-239 184 184 184 323 323 323 F F F
231402023-12-10T18:58:57.803Z 12 240-259 184 184 184 323 323 323 F F F
231412023-12-10T18:58:57.803Z 13 260-279 188 188 183 329 329 321 F F T <---
231422023-12-10T18:58:57.803Z 14 280-299 188 188 183 329 329 321 F F T <---
231432023-12-10T18:58:57.803Z 15 300-319 182 182 182 320 320 320 F F F
231442023-12-10T18:58:57.803Z 16 320-339 188 188 187 331 331 327 F F F <---
231452023-12-10T18:58:57.803Z 17 340-359 188 188 184 329 329 323 F F T <---
231462023-12-10T18:58:57.803Z 18 360-379 188 188 188 329 329 328 F F T <---
231472023-12-10T18:58:57.803Z 19 380-399 188 188 188 328 328 328 F F F
231482023-12-10T18:58:57.803Z 20 400-419 186 186 186 326 326 326 F F F
231492023-12-10T18:58:57.803Z 21 420-439 187 187 187 327 327 327 F F F
231502023-12-10T18:58:57.803Z 22 440-459 186 186 186 325 325 325 F F F
231512023-12-10T18:58:57.803Z 23 460-479 185 185 185 324 324 324 T T F <---
231522023-12-10T18:58:57.803Z 24 480-499 187 187 187 327 327 327 F F F
231532023-12-10T18:58:57.803Z 25 500-519 187 187 187 327 327 327 F F F
231542023-12-10T18:58:57.804Z 26 520-539 183 183 183 321 321 321 F F F
231552023-12-10T18:58:57.804Z 27 540-559 188 188 186 329 329 326 F F T <---
231562023-12-10T18:58:57.804Z 28 560-579 187 187 187 327 327 327 F F F
231572023-12-10T18:58:57.804Z 29 580-599 187 187 187 327 327 327 F F F
231582023-12-10T18:58:57.804ZMax gen: 188, Max flush: 331
231592023-12-10T18:58:57.804ZError: Difference in extent metadata found!
231602023-12-10T18:58:57.808ZA Difference in extent metadata is expected here
231612023-12-10T18:58:57.808ZOn loop 94
231622023-12-10T18:58:57.808Z
231632023-12-10T18:58:57.808ZVerifying data now
231642023-12-10T18:59:03.979ZNo change after verify
231652023-12-10T18:59:03.979ZLoop: 94 Downstairs dump after verify (and repair):
231662023-12-10T18:59:04.002ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
231672023-12-10T18:59:04.002Z 0 000-019 183 183 183 321 321 321 F F F
231682023-12-10T18:59:04.006Z 1 020-039 185 185 185 324 324 324 F F F
231692023-12-10T18:59:04.006Z 2 040-059 188 188 188 329 329 329 F F F
231702023-12-10T18:59:04.006Z 3 060-079 188 188 188 329 329 329 F F F
231712023-12-10T18:59:04.007Z 4 080-099 185 185 185 324 324 324 F F F
231722023-12-10T18:59:04.007Z 5 100-119 188 188 188 329 329 329 F F F
231732023-12-10T18:59:04.007Z 6 120-139 189 189 189 332 332 332 F F F
231742023-12-10T18:59:04.007Z 7 140-159 189 189 189 332 332 332 F F F
231752023-12-10T18:59:04.007Z 8 160-179 187 187 187 327 327 327 F F F
231762023-12-10T18:59:04.007Z 9 180-199 187 187 187 327 327 327 F F F
231772023-12-10T18:59:04.007Z 10 200-219 185 185 185 324 324 324 F F F
231782023-12-10T18:59:04.007Z 11 220-239 184 184 184 323 323 323 F F F
231792023-12-10T18:59:04.008Z 12 240-259 184 184 184 323 323 323 F F F
231802023-12-10T18:59:04.008Z 13 260-279 188 188 188 329 329 329 F F F
231812023-12-10T18:59:04.008Z 14 280-299 188 188 188 329 329 329 F F F
231822023-12-10T18:59:04.008Z 15 300-319 182 182 182 320 320 320 F F F
231832023-12-10T18:59:04.008Z 16 320-339 188 188 188 331 331 331 F F F
231842023-12-10T18:59:04.008Z 17 340-359 188 188 188 329 329 329 F F F
231852023-12-10T18:59:04.008Z 18 360-379 188 188 188 329 329 329 F F F
231862023-12-10T18:59:04.008Z 19 380-399 188 188 188 328 328 328 F F F
231872023-12-10T18:59:04.009Z 20 400-419 186 186 186 326 326 326 F F F
231882023-12-10T18:59:04.009Z 21 420-439 187 187 187 327 327 327 F F F
231892023-12-10T18:59:04.009Z 22 440-459 186 186 186 325 325 325 F F F
231902023-12-10T18:59:04.009Z 23 460-479 189 189 189 332 332 332 F F F
231912023-12-10T18:59:04.009Z 24 480-499 187 187 187 327 327 327 F F F
231922023-12-10T18:59:04.009Z 25 500-519 187 187 187 327 327 327 F F F
231932023-12-10T18:59:04.009Z 26 520-539 183 183 183 321 321 321 F F F
231942023-12-10T18:59:04.009Z 27 540-559 188 188 188 329 329 329 F F F
231952023-12-10T18:59:04.009Z 28 560-579 187 187 187 327 327 327 F F F
231962023-12-10T18:59:04.009Z 29 580-599 187 187 187 327 327 327 F F F
231972023-12-10T18:59:04.009ZMax gen: 189, Max flush: 332
231982023-12-10T18:59:04.012Z
231992023-12-10T18:59:04.012ZBegin loop 95
232002023-12-10T18:59:04.013ZDownstairs to restart: 1
232012023-12-10T18:59:04.039Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.036735467Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922}
232022023-12-10T18:59:04.039Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.037130625Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922}
232032023-12-10T18:59:04.039Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.037158836Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922}
232042023-12-10T18:59:04.045Z{"msg":"Crucible 8d52e90e-8812-446d-8e07-44b601b68792 has session id: f91c075b-48cf-429c-ae0f-2200c9d5046e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.037203092Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232052023-12-10T18:59:04.045Z{"msg":"Upstairs opts: Upstairs UUID: 8d52e90e-8812-446d-8e07-44b601b68792, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.037285108Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232062023-12-10T18:59:04.046Z{"msg":"Crucible stats registered with UUID: 8d52e90e-8812-446d-8e07-44b601b68792","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.037332294Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"Crucible runtime is spawned
232072023-12-10T18:59:04.046Zf91c075b-48cf-429c-ae0f-2200c9d5046e"}
232082023-12-10T18:59:04.046ZThe guest has requested activation with gen:190
232092023-12-10T18:59:04.046Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.037593232Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"looper":"2","session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232102023-12-10T18:59:04.046Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.037679401Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e","task":"up_listen"}
232112023-12-10T18:59:04.046Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.037709148Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232122023-12-10T18:59:04.046Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.037759782Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232132023-12-10T18:59:04.047Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.037928027Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"looper":"0","session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232142023-12-10T18:59:04.047Z{"msg":"[2] 8d52e90e-8812-446d-8e07-44b601b68792 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.037995885Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"looper":"2","session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232152023-12-10T18:59:04.047Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.038033194Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"looper":"1","session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232162023-12-10T18:59:04.047Z{"msg":"[0] 8d52e90e-8812-446d-8e07-44b601b68792 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.038084243Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"looper":"0","session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232172023-12-10T18:59:04.047Z{"msg":"[1] 8d52e90e-8812-446d-8e07-44b601b68792 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.038295859Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"looper":"1","session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232182023-12-10T18:59:04.047Z{"msg":"8d52e90e-8812-446d-8e07-44b601b68792 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.038659198Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232192023-12-10T18:59:04.047Z{"msg":"Set desired generation to :190","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.038810187Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232202023-12-10T18:59:04.047Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.038872551Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232212023-12-10T18:59:04.047Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.039116986Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232222023-12-10T18:59:04.047Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.0393783Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232232023-12-10T18:59:04.047Z{"msg":"[2] 8d52e90e-8812-446d-8e07-44b601b68792 (f91c075b-48cf-429c-ae0f-2200c9d5046e) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.039556261Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232242023-12-10T18:59:04.047Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.039694491Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232252023-12-10T18:59:04.047Z{"msg":"[2] client is_active_req TRUE, promote! session f91c075b-48cf-429c-ae0f-2200c9d5046e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.039740252Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232262023-12-10T18:59:04.048Z{"msg":"[0] 8d52e90e-8812-446d-8e07-44b601b68792 (f91c075b-48cf-429c-ae0f-2200c9d5046e) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.039814884Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232272023-12-10T18:59:04.048Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.039848642Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232282023-12-10T18:59:04.048Z{"msg":"[0] client is_active_req TRUE, promote! session f91c075b-48cf-429c-ae0f-2200c9d5046e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.039925298Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232292023-12-10T18:59:04.048Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.040318794Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232302023-12-10T18:59:04.048Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.040377223Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232312023-12-10T18:59:04.048Z{"msg":"8d52e90e-8812-446d-8e07-44b601b68792 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.040456989Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232322023-12-10T18:59:04.048Z{"msg":"[1] 8d52e90e-8812-446d-8e07-44b601b68792 (f91c075b-48cf-429c-ae0f-2200c9d5046e) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.040607269Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232332023-12-10T18:59:04.050Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.04073133Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232342023-12-10T18:59:04.050Z{"msg":"[1] client is_active_req TRUE, promote! session f91c075b-48cf-429c-ae0f-2200c9d5046e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.040849942Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232352023-12-10T18:59:04.050Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.040909964Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232362023-12-10T18:59:04.050Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.040957959Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232372023-12-10T18:59:04.050Z{"msg":"8d52e90e-8812-446d-8e07-44b601b68792 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.041050945Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232382023-12-10T18:59:04.056Z{"msg":"[0] 8d52e90e-8812-446d-8e07-44b601b68792 (f91c075b-48cf-429c-ae0f-2200c9d5046e) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.041265443Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232392023-12-10T18:59:04.056Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.041371673Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232402023-12-10T18:59:04.056Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:59:04.041442514Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232412023-12-10T18:59:04.056Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.041485249Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232422023-12-10T18:59:04.057Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.041554626Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232432023-12-10T18:59:04.057Z{"msg":"8d52e90e-8812-446d-8e07-44b601b68792 WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.041654487Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232442023-12-10T18:59:04.057Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.041696469Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232452023-12-10T18:59:04.057Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.041732925Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232462023-12-10T18:59:04.057Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.041860836Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232472023-12-10T18:59:04.057Z{"msg":"[2] 8d52e90e-8812-446d-8e07-44b601b68792 (f91c075b-48cf-429c-ae0f-2200c9d5046e) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.041927707Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232482023-12-10T18:59:04.057Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.041953562Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232492023-12-10T18:59:04.057Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.042006491Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232502023-12-10T18:59:04.057Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:59:04.042029768Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232512023-12-10T18:59:04.057Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.04207133Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232522023-12-10T18:59:04.057Z{"msg":"8d52e90e-8812-446d-8e07-44b601b68792 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.042095691Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232532023-12-10T18:59:04.057Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.042137294Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232542023-12-10T18:59:04.057Z{"msg":"8d52e90e-8812-446d-8e07-44b601b68792 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.042160669Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232552023-12-10T18:59:04.057Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.042201324Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232562023-12-10T18:59:04.057Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.042223456Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232572023-12-10T18:59:04.057Z{"msg":"[1] 8d52e90e-8812-446d-8e07-44b601b68792 (f91c075b-48cf-429c-ae0f-2200c9d5046e) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.042533668Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232582023-12-10T18:59:04.058Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.042592728Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232592023-12-10T18:59:04.058Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:59:04.042617529Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232602023-12-10T18:59:04.058Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.04267542Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232612023-12-10T18:59:04.058Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.042699812Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232622023-12-10T18:59:04.058Z{"msg":"8d52e90e-8812-446d-8e07-44b601b68792 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.042721452Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232632023-12-10T18:59:04.058Z{"msg":"[0]R flush_numbers[0..12]: [321, 324, 329, 329, 324, 329, 332, 332, 327, 327, 324, 323]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.042745943Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232642023-12-10T18:59:04.059Z{"msg":"[0]R generation[0..12]: [183, 185, 188, 188, 185, 188, 189, 189, 187, 187, 185, 184]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.042768515Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232652023-12-10T18:59:04.059Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.04281163Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232662023-12-10T18:59:04.059Z{"msg":"[1]R flush_numbers[0..12]: [321, 324, 329, 329, 324, 329, 332, 332, 327, 327, 324, 323]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.04283616Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232672023-12-10T18:59:04.059Z{"msg":"[1]R generation[0..12]: [183, 185, 188, 188, 185, 188, 189, 189, 187, 187, 185, 184]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.042858045Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232682023-12-10T18:59:04.059Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.042888395Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232692023-12-10T18:59:04.059Z{"msg":"The guest has finished waiting for activation with:190
232702023-12-10T18:59:04.059Z[2]R flush_numbers[0..12]: [321, 324, 329, 329, 324, 329, 332, 332, 327, 327, 324, 323]","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO
232712023-12-10T18:59:04.059Z,"level":30,"time":"2023-12-10T18:59:04.042977355Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232722023-12-10T18:59:04.059Z{"msg":"[2]R generation[0..12]: [183, 185, 188, 188, 185, 188, 189, 189, 187, 187, 185, 184]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.043016284Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232732023-12-10T18:59:04.059Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.043058606Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232742023-12-10T18:59:04.059Z{"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.043135063Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232752023-12-10T18:59:04.059Z{"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.043212851Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232762023-12-10T18:59:04.060Z{"msg":"Next flush: 333","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.04323833Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232772023-12-10T18:59:04.060Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.043280336Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232782023-12-10T18:59:04.060Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.043302871Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232792023-12-10T18:59:04.060Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.043364808Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232802023-12-10T18:59:04.060Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.043389714Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232812023-12-10T18:59:04.060Z{"msg":"8d52e90e-8812-446d-8e07-44b601b68792 is now active with session: f91c075b-48cf-429c-ae0f-2200c9d5046e","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.043449041Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232822023-12-10T18:59:04.060Z{"msg":"8d52e90e-8812-446d-8e07-44b601b68792 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.04347282Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232832023-12-10T18:59:04.060Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.043493783Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232842023-12-10T18:59:04.060Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.043534295Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232852023-12-10T18:59:04.060Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.043556957Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232862023-12-10T18:59:04.061Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.043577273Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232872023-12-10T18:59:04.061Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.043630279Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232882023-12-10T18:59:04.061Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.043652553Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232892023-12-10T18:59:04.061Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.04376268Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232902023-12-10T18:59:04.061Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.043895525Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232912023-12-10T18:59:04.061Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.043939906Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232922023-12-10T18:59:04.061Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.044007056Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232932023-12-10T18:59:04.061Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:04.044045569Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1922,"session_id":"f91c075b-48cf-429c-ae0f-2200c9d5046e"}
232942023-12-10T18:59:04.061ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
232952023-12-10T18:59:04.061ZLoading write count information from file "/tmp/test_repair_verify.data"
232962023-12-10T18:59:04.061ZRun Repair workload
232972023-12-10T18:59:04.061Z01/30 Write block 337 len 16384 data: 15 13 16 15
232982023-12-10T18:59:04.061Z02/30 Write block 175 len 4096 data: 12
232992023-12-10T18:59:04.061Z03/30 Write block 48 len 36864 data: 11 13 13 12 11 11 13 12 13
233002023-12-10T18:59:04.062Z04/30 Read block 249 len 20480
233012023-12-10T18:59:04.062Z05/30 Read block 337 len 8192
233022023-12-10T18:59:04.062Z06/30 Write block 489 len 8192 data: 15 13
233032023-12-10T18:59:04.065Z07/30 Write block 40 len 16384 data: 11 10 10 10
233042023-12-10T18:59:04.065Z08/30 Write block 165 len 24576 data: 17 19 18 18 17 17
233052023-12-10T18:59:04.066Z09/30 Read block 217 len 16384
233062023-12-10T18:59:04.066Z10/30 Read block 197 len 20480
233072023-12-10T18:59:04.066Z11/30 Read block 56 len 8192
233082023-12-10T18:59:04.066Z12/30 Read block 358 len 4096
233092023-12-10T18:59:04.066Z13/30 Write block 414 len 32768 data: 12 12 13 14 15 14 13 15
233102023-12-10T18:59:04.066Z14/30 Flush
233112023-12-10T18:59:04.069Z15/30 Flush
233122023-12-10T18:59:04.069Z16/30 Write block 522 len 20480 data: 13 15 16 16 18
233132023-12-10T18:59:04.072Z17/30 Read block 338 len 40960
233142023-12-10T18:59:04.075Z18/30 Read block 570 len 16384
233152023-12-10T18:59:04.075Z19/30 Read block 204 len 40960
233162023-12-10T18:59:04.078Z20/30 Read block 297 len 16384
233172023-12-10T18:59:04.082Z21/30 Read block 328 len 40960
233182023-12-10T18:59:04.085Z22/30 Read block 259 len 36864
233192023-12-10T18:59:04.091Z23/30 Write block 203 len 16384 data: 19 16 15 11
233202023-12-10T18:59:04.091Z24/30 Write block 252 len 28672 data: 16 15 14 12 12 10 10
233212023-12-10T18:59:04.091Z25/30 Flush
233222023-12-10T18:59:04.094Z26/30 Write block 308 len 8192 data: 12 11
233232023-12-10T18:59:04.094Z27/30 Read block 255 len 24576
233242023-12-10T18:59:04.094Z28/30 Write block 3 len 24576 data: 7 6 7 7 11 15
233252023-12-10T18:59:04.097Z29/30 Read block 422 len 24576
233262023-12-10T18:59:04.099Z30/30 Write block 39 len 12288 data: 12 12 11
233272023-12-10T18:59:04.103Z----------------------------------------------------------------
233282023-12-10T18:59:04.103Z Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:30
233292023-12-10T18:59:04.103ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
233302023-12-10T18:59:04.103Z 1 Acked 1000 Write 4 Done Done Done false
233312023-12-10T18:59:04.104Z 2 Acked 1001 Write 1 Done Done Done false
233322023-12-10T18:59:04.104Z 3 Acked 1002 Write 9 Done Done Done false
233332023-12-10T18:59:04.104Z 4 Acked 1003 Read 5 Done Sent Done false
233342023-12-10T18:59:04.104Z 5 Acked 1004 Read 2 Done Sent Done false
233352023-12-10T18:59:04.104Z 6 Acked 1005 Write 2 Done Sent Done false
233362023-12-10T18:59:04.104Z 7 Acked 1006 Write 4 Done Sent Done false
233372023-12-10T18:59:04.104Z 8 Acked 1007 Write 6 Done Sent Done false
233382023-12-10T18:59:04.104Z 9 Acked 1008 Read 4 Done Sent Done false
233392023-12-10T18:59:04.104Z 10 Acked 1009 Read 5 Done Sent Done false
233402023-12-10T18:59:04.104Z 11 Acked 1010 Read 2 Done Sent Done false
233412023-12-10T18:59:04.104Z 12 Acked 1011 Read 1 Done Sent Done false
233422023-12-10T18:59:04.104Z 13 Acked 1012 Write 8 Done Sent Done false
233432023-12-10T18:59:04.104Z 14 Acked 1013 Flush 0 Done Sent Done false
233442023-12-10T18:59:04.104Z 15 Acked 1014 Flush 0 Done Sent Done false
233452023-12-10T18:59:04.104Z 16 Acked 1015 Write 5 Done Sent Done false
233462023-12-10T18:59:04.104Z 17 Acked 1016 Read 10 Done Sent Done false
233472023-12-10T18:59:04.105Z 18 Acked 1017 Read 4 Done Sent Done false
233482023-12-10T18:59:04.105Z 19 Acked 1018 Read 10 Done Sent Done false
233492023-12-10T18:59:04.105Z 20 Acked 1019 Read 4 Done Sent Done false
233502023-12-10T18:59:04.105Z 21 Acked 1020 Read 10 Done Sent Done false
233512023-12-10T18:59:04.105Z 22 Acked 1021 Read 9 Done Sent Done false
233522023-12-10T18:59:04.105Z 23 Acked 1022 Write 4 Done Sent Done false
233532023-12-10T18:59:04.105Z 24 Acked 1023 Write 7 Done Sent Done false
233542023-12-10T18:59:04.105Z 25 Acked 1024 Flush 0 Done Sent Done false
233552023-12-10T18:59:04.105Z 26 Acked 1025 Write 2 Done Sent Done false
233562023-12-10T18:59:04.105Z 27 Acked 1026 Read 6 Done Sent Done false
233572023-12-10T18:59:04.105Z 28 Acked 1027 Write 6 Done Sent Done false
233582023-12-10T18:59:04.105Z 29 Acked 1028 Read 6 Done Sent Done false
233592023-12-10T18:59:04.105Z 30 Acked 1029 Write 3 Sent Sent Sent false
233602023-12-10T18:59:04.105Z STATES DS:0 DS:1 DS:2 TOTAL
233612023-12-10T18:59:04.105Z New 0 0 0 0
233622023-12-10T18:59:04.105Z Sent 1 27 1 29
233632023-12-10T18:59:04.105Z Done 29 3 29 61
233642023-12-10T18:59:04.105Z Skipped 0 0 0 0
233652023-12-10T18:59:04.106Z Error 0 0 0 0
233662023-12-10T18:59:04.106ZLast Flush: 1024 0 1024
233672023-12-10T18:59:04.106ZDownstairs last five completed:
233682023-12-10T18:59:04.106ZUpstairs last five completed: 30 29 28 27 26 25
233692023-12-10T18:59:04.106ZWrote out file "/tmp/test_repair_verify.data"
233702023-12-10T18:59:04.110Z
233712023-12-10T18:59:04.113ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
233722023-12-10T18:59:04.139ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
233732023-12-10T18:59:04.142Z 0 000-019 183 183 183 321 321 321 T F T <---
233742023-12-10T18:59:04.142Z 1 020-039 185 185 185 324 324 324 T F T <---
233752023-12-10T18:59:04.142Z 2 040-059 190 188 190 333 329 333 T T T <---
233762023-12-10T18:59:04.142Z 3 060-079 188 188 188 329 329 329 F F F
233772023-12-10T18:59:04.146Z 4 080-099 185 185 185 324 324 324 F F F
233782023-12-10T18:59:04.146Z 5 100-119 188 188 188 329 329 329 F F F
233792023-12-10T18:59:04.146Z 6 120-139 189 189 189 332 332 332 F F F
233802023-12-10T18:59:04.146Z 7 140-159 189 189 189 332 332 332 F F F
233812023-12-10T18:59:04.146Z 8 160-179 190 187 190 333 327 333 F T F <---
233822023-12-10T18:59:04.146Z 9 180-199 187 187 187 327 327 327 F F F
233832023-12-10T18:59:04.146Z 10 200-219 190 185 190 335 324 335 F F F <---
233842023-12-10T18:59:04.147Z 11 220-239 184 184 184 323 323 323 F F F
233852023-12-10T18:59:04.147Z 12 240-259 190 184 190 335 323 335 F F F <---
233862023-12-10T18:59:04.147Z 13 260-279 188 188 188 329 329 329 F F F
233872023-12-10T18:59:04.147Z 14 280-299 188 188 188 329 329 329 F F F
233882023-12-10T18:59:04.147Z 15 300-319 182 182 182 320 320 320 T F T <---
233892023-12-10T18:59:04.147Z 16 320-339 190 188 190 333 331 333 F T F <---
233902023-12-10T18:59:04.147Z 17 340-359 190 188 190 333 329 333 F T F <---
233912023-12-10T18:59:04.147Z 18 360-379 188 188 188 329 329 329 F F F
233922023-12-10T18:59:04.147Z 19 380-399 188 188 188 328 328 328 F F F
233932023-12-10T18:59:04.147Z 20 400-419 190 186 190 333 326 333 F F F <---
233942023-12-10T18:59:04.147Z 21 420-439 190 187 190 333 327 333 F F F <---
233952023-12-10T18:59:04.147Z 22 440-459 186 186 186 325 325 325 F F F
233962023-12-10T18:59:04.147Z 23 460-479 189 189 189 332 332 332 F F F
233972023-12-10T18:59:04.148Z 24 480-499 190 187 190 333 327 333 F F F <---
233982023-12-10T18:59:04.148Z 25 500-519 187 187 187 327 327 327 F F F
233992023-12-10T18:59:04.148Z 26 520-539 190 183 190 335 321 335 F F F <---
234002023-12-10T18:59:04.148Z 27 540-559 188 188 188 329 329 329 F F F
234012023-12-10T18:59:04.148Z 28 560-579 187 187 187 327 327 327 F F F
234022023-12-10T18:59:04.148Z 29 580-599 187 187 187 327 327 327 F F F
234032023-12-10T18:59:04.148ZMax gen: 190, Max flush: 335
234042023-12-10T18:59:04.148ZError: Difference in extent metadata found!
234052023-12-10T18:59:04.148ZA Difference in extent metadata is expected here
234062023-12-10T18:59:04.148ZOn loop 95
234072023-12-10T18:59:04.148Z
234082023-12-10T18:59:04.153ZVerifying data now
234092023-12-10T18:59:10.409ZNo change after verify
234102023-12-10T18:59:10.409ZLoop: 95 Downstairs dump after verify (and repair):
234112023-12-10T18:59:10.429ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
234122023-12-10T18:59:10.429Z 0 000-019 191 191 191 336 336 336 F F F
234132023-12-10T18:59:10.429Z 1 020-039 191 191 191 336 336 336 F F F
234142023-12-10T18:59:10.429Z 2 040-059 191 191 191 336 336 336 F F F
234152023-12-10T18:59:10.429Z 3 060-079 188 188 188 329 329 329 F F F
234162023-12-10T18:59:10.429Z 4 080-099 185 185 185 324 324 324 F F F
234172023-12-10T18:59:10.429Z 5 100-119 188 188 188 329 329 329 F F F
234182023-12-10T18:59:10.430Z 6 120-139 189 189 189 332 332 332 F F F
234192023-12-10T18:59:10.430Z 7 140-159 189 189 189 332 332 332 F F F
234202023-12-10T18:59:10.433Z 8 160-179 190 190 190 333 333 333 F F F
234212023-12-10T18:59:10.433Z 9 180-199 187 187 187 327 327 327 F F F
234222023-12-10T18:59:10.434Z 10 200-219 190 190 190 335 335 335 F F F
234232023-12-10T18:59:10.434Z 11 220-239 184 184 184 323 323 323 F F F
234242023-12-10T18:59:10.435Z 12 240-259 190 190 190 335 335 335 F F F
234252023-12-10T18:59:10.435Z 13 260-279 188 188 188 329 329 329 F F F
234262023-12-10T18:59:10.435Z 14 280-299 188 188 188 329 329 329 F F F
234272023-12-10T18:59:10.435Z 15 300-319 191 191 191 336 336 336 F F F
234282023-12-10T18:59:10.435Z 16 320-339 190 190 190 333 333 333 F F F
234292023-12-10T18:59:10.435Z 17 340-359 190 190 190 333 333 333 F F F
234302023-12-10T18:59:10.435Z 18 360-379 188 188 188 329 329 329 F F F
234312023-12-10T18:59:10.435Z 19 380-399 188 188 188 328 328 328 F F F
234322023-12-10T18:59:10.435Z 20 400-419 190 190 190 333 333 333 F F F
234332023-12-10T18:59:10.435Z 21 420-439 190 190 190 333 333 333 F F F
234342023-12-10T18:59:10.436Z 22 440-459 186 186 186 325 325 325 F F F
234352023-12-10T18:59:10.436Z 23 460-479 189 189 189 332 332 332 F F F
234362023-12-10T18:59:10.436Z 24 480-499 190 190 190 333 333 333 F F F
234372023-12-10T18:59:10.436Z 25 500-519 187 187 187 327 327 327 F F F
234382023-12-10T18:59:10.436Z 26 520-539 190 190 190 335 335 335 F F F
234392023-12-10T18:59:10.436Z 27 540-559 188 188 188 329 329 329 F F F
234402023-12-10T18:59:10.436Z 28 560-579 187 187 187 327 327 327 F F F
234412023-12-10T18:59:10.436Z 29 580-599 187 187 187 327 327 327 F F F
234422023-12-10T18:59:10.436ZMax gen: 191, Max flush: 336
234432023-12-10T18:59:10.436Z
234442023-12-10T18:59:10.437ZBegin loop 96
234452023-12-10T18:59:10.437ZDownstairs to restart: 1
234462023-12-10T18:59:10.463Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.461416926Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930}
234472023-12-10T18:59:10.463Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.461802103Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930}
234482023-12-10T18:59:10.463Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.4618531Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930}
234492023-12-10T18:59:10.469Z{"msg":"Crucible 181ef3d9-3d70-4480-88c7-ce1a74213f7d has session id: 7f21c51a-5607-423f-8381-b0ac300cd995","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.461882429Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234502023-12-10T18:59:10.469Z{"msg":"Upstairs opts: Upstairs UUID: 181ef3d9-3d70-4480-88c7-ce1a74213f7d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.461975086Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234512023-12-10T18:59:10.469Z{"msg":"Crucible stats registered with UUID: 181ef3d9-3d70-4480-88c7-ce1a74213f7d","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.4620501Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234522023-12-10T18:59:10.469Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30Crucible runtime is spawned
234532023-12-10T18:59:10.469Z,"time":"2023-12-10T18:59:10.462212338Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995","task":"up_listen"}
234542023-12-10T18:59:10.470Z{"msg":"[0] connecting to 127.0.0.1:8810","v":The guest has requested activation with gen:192
234552023-12-10T18:59:10.470Z0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.462499526Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"looper":"0","session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234562023-12-10T18:59:10.470Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.462737811Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234572023-12-10T18:59:10.470Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.462825028Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234582023-12-10T18:59:10.470Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.46286775Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"looper":"1","session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234592023-12-10T18:59:10.470Z{"msg":"[0] 181ef3d9-3d70-4480-88c7-ce1a74213f7d looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.462933925Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"looper":"0","session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234602023-12-10T18:59:10.470Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.46297514Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"looper":"2","session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234612023-12-10T18:59:10.470Z{"msg":"[1] 181ef3d9-3d70-4480-88c7-ce1a74213f7d looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.463078265Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"looper":"1","session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234622023-12-10T18:59:10.470Z{"msg":"[2] 181ef3d9-3d70-4480-88c7-ce1a74213f7d looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.463121103Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"looper":"2","session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234632023-12-10T18:59:10.470Z{"msg":"181ef3d9-3d70-4480-88c7-ce1a74213f7d active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.463196426Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234642023-12-10T18:59:10.471Z{"msg":"Set desired generation to :192","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.463235192Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234652023-12-10T18:59:10.471Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.463275328Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234662023-12-10T18:59:10.471Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.463652549Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234672023-12-10T18:59:10.471Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.463778572Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234682023-12-10T18:59:10.471Z{"msg":"[0] 181ef3d9-3d70-4480-88c7-ce1a74213f7d (7f21c51a-5607-423f-8381-b0ac300cd995) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.464011629Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234692023-12-10T18:59:10.471Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.464081567Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234702023-12-10T18:59:10.471Z{"msg":"[0] client is_active_req TRUE, promote! session 7f21c51a-5607-423f-8381-b0ac300cd995","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.464108912Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234712023-12-10T18:59:10.471Z{"msg":"[2] 181ef3d9-3d70-4480-88c7-ce1a74213f7d (7f21c51a-5607-423f-8381-b0ac300cd995) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.464270685Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234722023-12-10T18:59:10.471Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.46439407Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234732023-12-10T18:59:10.471Z{"msg":"[2] client is_active_req TRUE, promote! session 7f21c51a-5607-423f-8381-b0ac300cd995","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.464479111Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234742023-12-10T18:59:10.472Z{"msg":"[1] 181ef3d9-3d70-4480-88c7-ce1a74213f7d (7f21c51a-5607-423f-8381-b0ac300cd995) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.46456121Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234752023-12-10T18:59:10.472Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.464606075Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234762023-12-10T18:59:10.472Z{"msg":"[1] client is_active_req TRUE, promote! session 7f21c51a-5607-423f-8381-b0ac300cd995","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.464682675Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234772023-12-10T18:59:10.472Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.464963261Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234782023-12-10T18:59:10.472Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.465068892Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234792023-12-10T18:59:10.472Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.465106986Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234802023-12-10T18:59:10.472Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.465153063Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234812023-12-10T18:59:10.472Z{"msg":"181ef3d9-3d70-4480-88c7-ce1a74213f7d WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.465220776Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234822023-12-10T18:59:10.472Z{"msg":"181ef3d9-3d70-4480-88c7-ce1a74213f7d WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.465262268Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234832023-12-10T18:59:10.473Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.465468345Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234842023-12-10T18:59:10.473Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.46557148Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234852023-12-10T18:59:10.473Z{"msg":"181ef3d9-3d70-4480-88c7-ce1a74213f7d WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.465750098Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234862023-12-10T18:59:10.479Z{"msg":"[2] 181ef3d9-3d70-4480-88c7-ce1a74213f7d (7f21c51a-5607-423f-8381-b0ac300cd995) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.465799786Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234872023-12-10T18:59:10.479Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.465874956Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234882023-12-10T18:59:10.479Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:59:10.465951942Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234892023-12-10T18:59:10.479Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.46603155Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234902023-12-10T18:59:10.479Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.466109082Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234912023-12-10T18:59:10.480Z{"msg":"181ef3d9-3d70-4480-88c7-ce1a74213f7d WaitActive WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.466189496Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234922023-12-10T18:59:10.480Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.466224276Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234932023-12-10T18:59:10.480Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.466285109Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234942023-12-10T18:59:10.480Z{"msg":"[0] 181ef3d9-3d70-4480-88c7-ce1a74213f7d (7f21c51a-5607-423f-8381-b0ac300cd995) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.466333906Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234952023-12-10T18:59:10.480Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.46640752Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234962023-12-10T18:59:10.480Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:59:10.466492846Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234972023-12-10T18:59:10.480Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.466568481Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234982023-12-10T18:59:10.480Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.466609983Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
234992023-12-10T18:59:10.480Z{"msg":"181ef3d9-3d70-4480-88c7-ce1a74213f7d WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.466678595Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235002023-12-10T18:59:10.481Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.466746352Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235012023-12-10T18:59:10.481Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:192
235022023-12-10T18:59:10.481Z,"time":"Wait for a query_work_queue command to finish before sending IO
235032023-12-10T18:59:10.481Z2023-12-10T18:59:10.466813999Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235042023-12-10T18:59:10.481Z{"msg":"[1] 181ef3d9-3d70-4480-88c7-ce1a74213f7d (7f21c51a-5607-423f-8381-b0ac300cd995) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.466859446Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235052023-12-10T18:59:10.481Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.466929219Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235062023-12-10T18:59:10.481Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:59:10.467033399Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235072023-12-10T18:59:10.481Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.467075394Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235082023-12-10T18:59:10.481Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.467179018Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235092023-12-10T18:59:10.481Z{"msg":"181ef3d9-3d70-4480-88c7-ce1a74213f7d WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.467260034Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235102023-12-10T18:59:10.482Z{"msg":"[0]R flush_numbers[0..12]: [336, 336, 336, 329, 324, 329, 332, 332, 333, 327, 335, 323]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.467359507Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235112023-12-10T18:59:10.482Z{"msg":"[0]R generation[0..12]: [191, 191, 191, 188, 185, 188, 189, 189, 190, 187, 190, 184]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.467405396Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235122023-12-10T18:59:10.482Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.467447358Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235132023-12-10T18:59:10.482Z{"msg":"[1]R flush_numbers[0..12]: [336, 336, 336, 329, 324, 329, 332, 332, 333, 327, 335, 323]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.467489956Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235142023-12-10T18:59:10.482Z{"msg":"[1]R generation[0..12]: [191, 191, 191, 188, 185, 188, 189, 189, 190, 187, 190, 184]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.467561483Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235152023-12-10T18:59:10.482Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.467640245Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235162023-12-10T18:59:10.482Z{"msg":"[2]R flush_numbers[0..12]: [336, 336, 336, 329, 324, 329, 332, 332, 333, 327, 335, 323]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.467716032Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235172023-12-10T18:59:10.482Z{"msg":"[2]R generation[0..12]: [191, 191, 191, 188, 185, 188, 189, 189, 190, 187, 190, 184]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.467759987Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235182023-12-10T18:59:10.482Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.467831598Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235192023-12-10T18:59:10.483Z{"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.467899803Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235202023-12-10T18:59:10.483Z{"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.467939557Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235212023-12-10T18:59:10.483Z{"msg":"Next flush: 337","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.467988196Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235222023-12-10T18:59:10.483Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.468010981Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235232023-12-10T18:59:10.483Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
235242023-12-10T18:59:10.483ZNo downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.468123275Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235252023-12-10T18:59:10.483Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.468188526Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235262023-12-10T18:59:10.483Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.468212354Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235272023-12-10T18:59:10.483Z{"msg":"181ef3d9-3d70-4480-88c7-ce1a74213f7d is now active with session: 7f21c51a-5607-423f-8381-b0ac300cd995","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.468234431Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235282023-12-10T18:59:10.484Z{"msg":"181ef3d9-3d70-4480-88c7-ce1a74213f7d Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.468275642Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235292023-12-10T18:59:10.484Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.468298366Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235302023-12-10T18:59:10.484Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.468318277Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235312023-12-10T18:59:10.485Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.468358171Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235322023-12-10T18:59:10.485Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.468380208Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235332023-12-10T18:59:10.485Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.468400041Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235342023-12-10T18:59:10.485Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.46843989Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235352023-12-10T18:59:10.485Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.468461729Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235362023-12-10T18:59:10.485Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.468481668Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235372023-12-10T18:59:10.485Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.468501158Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235382023-12-10T18:59:10.485Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.468520839Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235392023-12-10T18:59:10.485Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:10.46854023Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1930,"session_id":"7f21c51a-5607-423f-8381-b0ac300cd995"}
235402023-12-10T18:59:10.485ZLoading write count information from file "/tmp/test_repair_verify.data"
235412023-12-10T18:59:10.485ZRun Repair workload
235422023-12-10T18:59:10.486Z01/30 Write block 0 len 4096 data: 5
235432023-12-10T18:59:10.486Z02/30 Write block 90 len 8192 data: 15 12
235442023-12-10T18:59:10.486Z03/30 Read block 460 len 4096
235452023-12-10T18:59:10.486Z04/30 Read block 384 len 20480
235462023-12-10T18:59:10.486Z05/30 Read block 57 len 20480
235472023-12-10T18:59:10.490Z06/30 Read block 15 len 16384
235482023-12-10T18:59:10.490Z07/30 Flush
235492023-12-10T18:59:10.490Z08/30 Flush
235502023-12-10T18:59:10.490Z09/30 Flush
235512023-12-10T18:59:10.490Z10/30 Write block 152 len 12288 data: 14 15 19
235522023-12-10T18:59:10.490Z11/30 Write block 454 len 32768 data: 13 12 12 10 12 10 10 10
235532023-12-10T18:59:10.490Z12/30 Read block 373 len 32768
235542023-12-10T18:59:10.490Z13/30 Read block 214 len 4096
235552023-12-10T18:59:10.490Z14/30 Write block 174 len 8192 data: 14 13
235562023-12-10T18:59:10.493Z15/30 Write block 88 len 8192 data: 22 20
235572023-12-10T18:59:10.493Z16/30 Write block 431 len 16384 data: 14 13 12 12
235582023-12-10T18:59:10.493Z17/30 Read block 171 len 32768
235592023-12-10T18:59:10.496Z18/30 Read block 81 len 16384
235602023-12-10T18:59:10.499Z19/30 Write block 407 len 20480 data: 16 19 18 16 13
235612023-12-10T18:59:10.499Z20/30 Read block 382 len 40960
235622023-12-10T18:59:10.502Z21/30 Write block 204 len 40960 data: 17 16 12 12 11 10 10 11 7 10
235632023-12-10T18:59:10.502Z22/30 Read block 5 len 12288
235642023-12-10T18:59:10.505Z23/30 Read block 239 len 12288
235652023-12-10T18:59:10.511Z24/30 Flush
235662023-12-10T18:59:10.514Z25/30 Write block 213 len 32768 data: 11 15 16 19 19 17 16 17
235672023-12-10T18:59:10.514Z26/30 Read block 434 len 28672
235682023-12-10T18:59:10.514Z27/30 Read block 46 len 8192
235692023-12-10T18:59:10.514Z28/30 Read block 266 len 16384
235702023-12-10T18:59:10.517Z29/30 Read block 482 len 32768
235712023-12-10T18:59:10.520Z30/30 Read block 306 len 32768
235722023-12-10T18:59:10.524Z----------------------------------------------------------------
235732023-12-10T18:59:10.524Z Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30
235742023-12-10T18:59:10.524ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
235752023-12-10T18:59:10.524Z 1 Acked 1000 Write 1 Done Done Done false
235762023-12-10T18:59:10.524Z 2 Acked 1001 Write 2 Done Done Done false
235772023-12-10T18:59:10.524Z 3 Acked 1002 Read 1 Done Done Done false
235782023-12-10T18:59:10.524Z 4 Acked 1003 Read 5 Done Done Done false
235792023-12-10T18:59:10.524Z 5 Acked 1004 Read 5 Done Sent Done false
235802023-12-10T18:59:10.524Z 6 Acked 1005 Read 4 Done Sent Done false
235812023-12-10T18:59:10.524Z 7 Acked 1006 Flush 0 Done Sent Done false
235822023-12-10T18:59:10.524Z 8 Acked 1007 Flush 0 Done Sent Done false
235832023-12-10T18:59:10.524Z 9 Acked 1008 Flush 0 Done Sent Done false
235842023-12-10T18:59:10.524Z 10 Acked 1009 Write 3 Done Sent Done false
235852023-12-10T18:59:10.524Z 11 Acked 1010 Write 8 Done Sent Done false
235862023-12-10T18:59:10.524Z 12 Acked 1011 Read 8 Done Sent Done false
235872023-12-10T18:59:10.525Z 13 Acked 1012 Read 1 Done Sent Done false
235882023-12-10T18:59:10.525Z 14 Acked 1013 Write 2 Done Sent Done false
235892023-12-10T18:59:10.525Z 15 Acked 1014 Write 2 Done Sent Done false
235902023-12-10T18:59:10.525Z 16 Acked 1015 Write 4 Done Sent Done false
235912023-12-10T18:59:10.525Z 17 Acked 1016 Read 8 Done Sent Done false
235922023-12-10T18:59:10.525Z 18 Acked 1017 Read 4 Done Sent Done false
235932023-12-10T18:59:10.525Z 19 Acked 1018 Write 5 Done Sent Done false
235942023-12-10T18:59:10.525Z 20 Acked 1019 Read 10 Done Sent Done false
235952023-12-10T18:59:10.525Z 21 Acked 1020 Write 10 Done Sent Done false
235962023-12-10T18:59:10.525Z 22 Acked 1021 Read 3 Done Sent Done false
235972023-12-10T18:59:10.525Z 23 Acked 1022 Read 3 Done Sent Done false
235982023-12-10T18:59:10.525Z 24 Acked 1023 Flush 0 Done Sent Done false
235992023-12-10T18:59:10.525Z 25 Acked 1024 Write 8 Done Sent Done false
236002023-12-10T18:59:10.525Z 26 Acked 1025 Read 7 Done Sent Done false
236012023-12-10T18:59:10.525Z 27 Acked 1026 Read 2 Done Sent Done false
236022023-12-10T18:59:10.525Z 28 Acked 1027 Read 4 Done Sent Done false
236032023-12-10T18:59:10.525Z 29 Acked 1028 Read 8 Done Sent Done false
236042023-12-10T18:59:10.526Z 30 Acked 1029 Read 8 Done Sent Done false
236052023-12-10T18:59:10.526Z STATES DS:0 DS:1 DS:2 TOTAL
236062023-12-10T18:59:10.526Z New 0 0 0 0
236072023-12-10T18:59:10.526Z Sent 0 26 0 26
236082023-12-10T18:59:10.526Z Done 30 4 30 64
236092023-12-10T18:59:10.526Z Skipped 0 0 0 0
236102023-12-10T18:59:10.526Z Error 0 0 0 0
236112023-12-10T18:59:10.526ZLast Flush: 1023 0 1023
236122023-12-10T18:59:10.526ZDownstairs last five completed:
236132023-12-10T18:59:10.526ZUpstairs last five completed: 30 29 28 27 26 25
236142023-12-10T18:59:10.526ZWrote out file "/tmp/test_repair_verify.data"
236152023-12-10T18:59:10.529Z
236162023-12-10T18:59:10.536ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
236172023-12-10T18:59:10.569ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
236182023-12-10T18:59:10.569Z 0 000-019 192 191 192 337 336 337 F T F <---
236192023-12-10T18:59:10.569Z 1 020-039 191 191 191 336 336 336 F F F
236202023-12-10T18:59:10.569Z 2 040-059 191 191 191 336 336 336 F F F
236212023-12-10T18:59:10.569Z 3 060-079 188 188 188 329 329 329 F F F
236222023-12-10T18:59:10.569Z 4 080-099 192 185 192 340 324 340 F T F <---
236232023-12-10T18:59:10.570Z 5 100-119 188 188 188 329 329 329 F F F
236242023-12-10T18:59:10.573Z 6 120-139 189 189 189 332 332 332 F F F
236252023-12-10T18:59:10.573Z 7 140-159 192 189 192 340 332 340 F F F <---
236262023-12-10T18:59:10.573Z 8 160-179 192 190 192 340 333 340 F F F <---
236272023-12-10T18:59:10.573Z 9 180-199 187 187 187 327 327 327 F F F
236282023-12-10T18:59:10.573Z 10 200-219 192 190 192 340 335 340 T F T <---
236292023-12-10T18:59:10.574Z 11 220-239 184 184 184 323 323 323 T F T <---
236302023-12-10T18:59:10.574Z 12 240-259 190 190 190 335 335 335 F F F
236312023-12-10T18:59:10.574Z 13 260-279 188 188 188 329 329 329 F F F
236322023-12-10T18:59:10.574Z 14 280-299 188 188 188 329 329 329 F F F
236332023-12-10T18:59:10.574Z 15 300-319 191 191 191 336 336 336 F F F
236342023-12-10T18:59:10.574Z 16 320-339 190 190 190 333 333 333 F F F
236352023-12-10T18:59:10.574Z 17 340-359 190 190 190 333 333 333 F F F
236362023-12-10T18:59:10.574Z 18 360-379 188 188 188 329 329 329 F F F
236372023-12-10T18:59:10.574Z 19 380-399 188 188 188 328 328 328 F F F
236382023-12-10T18:59:10.574Z 20 400-419 192 190 192 340 333 340 F F F <---
236392023-12-10T18:59:10.574Z 21 420-439 192 190 192 340 333 340 F F F <---
236402023-12-10T18:59:10.574Z 22 440-459 192 186 192 340 325 340 F F F <---
236412023-12-10T18:59:10.577Z 23 460-479 192 189 192 340 332 340 F F F <---
236422023-12-10T18:59:10.577Z 24 480-499 190 190 190 333 333 333 F F F
236432023-12-10T18:59:10.577Z 25 500-519 187 187 187 327 327 327 F F F
236442023-12-10T18:59:10.577Z 26 520-539 190 190 190 335 335 335 F F F
236452023-12-10T18:59:10.578Z 27 540-559 188 188 188 329 329 329 F F F
236462023-12-10T18:59:10.578Z 28 560-579 187 187 187 327 327 327 F F F
236472023-12-10T18:59:10.578Z 29 580-599 187 187 187 327 327 327 F F F
236482023-12-10T18:59:10.578ZMax gen: 192, Max flush: 340
236492023-12-10T18:59:10.578ZError: Difference in extent metadata found!
236502023-12-10T18:59:10.578ZA Difference in extent metadata is expected here
236512023-12-10T18:59:10.578ZOn loop 96
236522023-12-10T18:59:10.578Z
236532023-12-10T18:59:10.578ZVerifying data now
236542023-12-10T18:59:15.584ZNo change after verify
236552023-12-10T18:59:15.584ZLoop: 96 Downstairs dump after verify (and repair):
236562023-12-10T18:59:15.607ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
236572023-12-10T18:59:15.607Z 0 000-019 192 192 192 337 337 337 F F F
236582023-12-10T18:59:15.611Z 1 020-039 191 191 191 336 336 336 F F F
236592023-12-10T18:59:15.611Z 2 040-059 191 191 191 336 336 336 F F F
236602023-12-10T18:59:15.611Z 3 060-079 188 188 188 329 329 329 F F F
236612023-12-10T18:59:15.611Z 4 080-099 192 192 192 340 340 340 F F F
236622023-12-10T18:59:15.611Z 5 100-119 188 188 188 329 329 329 F F F
236632023-12-10T18:59:15.611Z 6 120-139 189 189 189 332 332 332 F F F
236642023-12-10T18:59:15.611Z 7 140-159 192 192 192 340 340 340 F F F
236652023-12-10T18:59:15.611Z 8 160-179 192 192 192 340 340 340 F F F
236662023-12-10T18:59:15.611Z 9 180-199 187 187 187 327 327 327 F F F
236672023-12-10T18:59:15.611Z 10 200-219 193 193 193 341 341 341 F F F
236682023-12-10T18:59:15.611Z 11 220-239 193 193 193 341 341 341 F F F
236692023-12-10T18:59:15.611Z 12 240-259 190 190 190 335 335 335 F F F
236702023-12-10T18:59:15.612Z 13 260-279 188 188 188 329 329 329 F F F
236712023-12-10T18:59:15.612Z 14 280-299 188 188 188 329 329 329 F F F
236722023-12-10T18:59:15.612Z 15 300-319 191 191 191 336 336 336 F F F
236732023-12-10T18:59:15.612Z 16 320-339 190 190 190 333 333 333 F F F
236742023-12-10T18:59:15.612Z 17 340-359 190 190 190 333 333 333 F F F
236752023-12-10T18:59:15.612Z 18 360-379 188 188 188 329 329 329 F F F
236762023-12-10T18:59:15.612Z 19 380-399 188 188 188 328 328 328 F F F
236772023-12-10T18:59:15.612Z 20 400-419 192 192 192 340 340 340 F F F
236782023-12-10T18:59:15.612Z 21 420-439 192 192 192 340 340 340 F F F
236792023-12-10T18:59:15.612Z 22 440-459 192 192 192 340 340 340 F F F
236802023-12-10T18:59:15.612Z 23 460-479 192 192 192 340 340 340 F F F
236812023-12-10T18:59:15.612Z 24 480-499 190 190 190 333 333 333 F F F
236822023-12-10T18:59:15.612Z 25 500-519 187 187 187 327 327 327 F F F
236832023-12-10T18:59:15.612Z 26 520-539 190 190 190 335 335 335 F F F
236842023-12-10T18:59:15.612Z 27 540-559 188 188 188 329 329 329 F F F
236852023-12-10T18:59:15.612Z 28 560-579 187 187 187 327 327 327 F F F
236862023-12-10T18:59:15.612Z 29 580-599 187 187 187 327 327 327 F F F
236872023-12-10T18:59:15.612ZMax gen: 193, Max flush: 341
236882023-12-10T18:59:15.616Z
236892023-12-10T18:59:15.616ZBegin loop 97
236902023-12-10T18:59:15.616ZDownstairs to restart: 1
236912023-12-10T18:59:15.641Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.639419786Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938}
236922023-12-10T18:59:15.641Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.639800544Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938}
236932023-12-10T18:59:15.646Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30Crucible runtime is spawned
236942023-12-10T18:59:15.646Z,"time":"2023-12-10T18:59:15.639904521Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938}
236952023-12-10T18:59:15.647Z{"msg":"The guest has requested activation with gen:194
236962023-12-10T18:59:15.647ZCrucible af130067-ef73-484b-8814-ffb2d94ba955 has session id: ddb8717d-39b9-428c-9a2f-46f8bc9de658","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.640277386Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
236972023-12-10T18:59:15.647Z{"msg":"Upstairs opts: Upstairs UUID: af130067-ef73-484b-8814-ffb2d94ba955, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.640351298Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
236982023-12-10T18:59:15.647Z{"msg":"Crucible stats registered with UUID: af130067-ef73-484b-8814-ffb2d94ba955","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.640378479Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
236992023-12-10T18:59:15.647Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.640398891Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658","task":"up_listen"}
237002023-12-10T18:59:15.647Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.640442621Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237012023-12-10T18:59:15.647Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.640465066Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237022023-12-10T18:59:15.647Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.640490603Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"looper":"0","session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237032023-12-10T18:59:15.647Z{"msg":"af130067-ef73-484b-8814-ffb2d94ba955 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.640515774Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237042023-12-10T18:59:15.647Z{"msg":"Set desired generation to :194","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.640540364Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237052023-12-10T18:59:15.647Z{"msg":"[0] af130067-ef73-484b-8814-ffb2d94ba955 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.640577157Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"looper":"0","session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237062023-12-10T18:59:15.647Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.640650173Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"looper":"2","session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237072023-12-10T18:59:15.647Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.640808312Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"looper":"1","session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237082023-12-10T18:59:15.647Z{"msg":"[2] af130067-ef73-484b-8814-ffb2d94ba955 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.640915265Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"looper":"2","session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237092023-12-10T18:59:15.647Z{"msg":"[1] af130067-ef73-484b-8814-ffb2d94ba955 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.641067149Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"looper":"1","session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237102023-12-10T18:59:15.647Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.641123321Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237112023-12-10T18:59:15.648Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.641187392Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237122023-12-10T18:59:15.648Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.641676664Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237132023-12-10T18:59:15.648Z{"msg":"[0] af130067-ef73-484b-8814-ffb2d94ba955 (ddb8717d-39b9-428c-9a2f-46f8bc9de658) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.642052384Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237142023-12-10T18:59:15.653Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.64213886Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237152023-12-10T18:59:15.653Z{"msg":"[0] client is_active_req TRUE, promote! session ddb8717d-39b9-428c-9a2f-46f8bc9de658","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.642267474Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237162023-12-10T18:59:15.653Z{"msg":"[2] af130067-ef73-484b-8814-ffb2d94ba955 (ddb8717d-39b9-428c-9a2f-46f8bc9de658) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.642502564Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237172023-12-10T18:59:15.653Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.642533617Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237182023-12-10T18:59:15.653Z{"msg":"[2] client is_active_req TRUE, promote! session ddb8717d-39b9-428c-9a2f-46f8bc9de658","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.642562727Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237192023-12-10T18:59:15.653Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.642906182Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237202023-12-10T18:59:15.653Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.642980085Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237212023-12-10T18:59:15.653Z{"msg":"af130067-ef73-484b-8814-ffb2d94ba955 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.643005685Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237222023-12-10T18:59:15.653Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.643137445Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237232023-12-10T18:59:15.653Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.643172172Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237242023-12-10T18:59:15.653Z{"msg":"af130067-ef73-484b-8814-ffb2d94ba955 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.643194553Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237252023-12-10T18:59:15.653Z{"msg":"[1] af130067-ef73-484b-8814-ffb2d94ba955 (ddb8717d-39b9-428c-9a2f-46f8bc9de658) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.643333853Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237262023-12-10T18:59:15.654Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.643393988Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237272023-12-10T18:59:15.660Z{"msg":"[1] client is_active_req TRUE, promote! session ddb8717d-39b9-428c-9a2f-46f8bc9de658","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.643462227Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237282023-12-10T18:59:15.660Z{"msg":"[0] af130067-ef73-484b-8814-ffb2d94ba955 (ddb8717d-39b9-428c-9a2f-46f8bc9de658) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.643698308Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237292023-12-10T18:59:15.660Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.643788314Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237302023-12-10T18:59:15.660Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:59:15.643814656Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237312023-12-10T18:59:15.660Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.643892229Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237322023-12-10T18:59:15.660Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.644008846Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237332023-12-10T18:59:15.660Z{"msg":"af130067-ef73-484b-8814-ffb2d94ba955 WaitQuorum WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.644103557Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237342023-12-10T18:59:15.660Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.64414612Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237352023-12-10T18:59:15.660Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.644217515Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237362023-12-10T18:59:15.660Z{"msg":"[2] af130067-ef73-484b-8814-ffb2d94ba955 (ddb8717d-39b9-428c-9a2f-46f8bc9de658) WaitQuorum WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.644261751Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237372023-12-10T18:59:15.660Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.644333369Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237382023-12-10T18:59:15.660Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:59:15.6443762Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237392023-12-10T18:59:15.660Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.644441769Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237402023-12-10T18:59:15.661Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.644469613Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237412023-12-10T18:59:15.661Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.644516216Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237422023-12-10T18:59:15.661Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.644539392Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237432023-12-10T18:59:15.661Z{"msg":"af130067-ef73-484b-8814-ffb2d94ba955 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.644579995Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237442023-12-10T18:59:15.661Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.644602491Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237452023-12-10T18:59:15.661Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.64462213Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237462023-12-10T18:59:15.661Z{"msg":"af130067-ef73-484b-8814-ffb2d94ba955 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.644643191Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237472023-12-10T18:59:15.661Z{"msg":"[1] af130067-ef73-484b-8814-ffb2d94ba955 (ddb8717d-39b9-428c-9a2f-46f8bc9de658) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.645026165Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237482023-12-10T18:59:15.661Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.645109004Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237492023-12-10T18:59:15.661Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:59:15.645181012Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237502023-12-10T18:59:15.661Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.645252095Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237512023-12-10T18:59:15.661Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.645290341Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237522023-12-10T18:59:15.661Z{"msg":"af130067-ef73-484b-8814-ffb2d94ba955 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.645357523Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237532023-12-10T18:59:15.661Z{"msg":"[0]R flush_numbers[0..12]: [337, 336, 336, 329, 340, 329, 332, 340, 340, 327, 341, 341]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.645430931Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237542023-12-10T18:59:15.663Z{"msg":"[0]R generation[0..12]: [192, 191, 191, 188, 192, 188, 189, 192, 192, 187, 193, 193]"The guest has finished waiting for activation with:194
237552023-12-10T18:59:15.663Z,"v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
237562023-12-10T18:59:15.663Z,"time":"2023-12-10T18:59:15.645534711Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237572023-12-10T18:59:15.663Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.645579578Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237582023-12-10T18:59:15.663Z{"msg":"[1]R flush_numbers[0..12]: [337, 336, 336, 329, 340, 329, 332, 340, 340, 327, 341, 341]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.645653035Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237592023-12-10T18:59:15.663Z{"msg":"[1]R generation[0..12]: [192, 191, 191, 188, 192, 188, 189, 192, 192, 187, 193, 193]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.645738223Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237602023-12-10T18:59:15.663Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.645780887Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237612023-12-10T18:59:15.663Z{"msg":"[2]R flush_numbers[0..12]: [337, 336, 336, 329, 340, 329, 332, 340, 340, 327, 341, 341]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.645831533Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237622023-12-10T18:59:15.663Z{"msg":"[2]R generation[0..12]: [192, 191, 191, 188, 192, 188, 189, 192, 192, 187, 193, 193]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.645855752Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237632023-12-10T18:59:15.663Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.645877743Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237642023-12-10T18:59:15.663Z{"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.645941332Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237652023-12-10T18:59:15.663Z{"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.645984221Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237662023-12-10T18:59:15.664Z{"msg":"Next flush: 342","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.646053283Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237672023-12-10T18:59:15.664Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.646124798Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237682023-12-10T18:59:15.664Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.646193842Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237692023-12-10T18:59:15.664Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.646262671Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237702023-12-10T18:59:15.664Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.646336584Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237712023-12-10T18:59:15.664Z{"msg":"af130067-ef73-484b-8814-ffb2d94ba955 is now active with session: ddb8717d-39b9-428c-9a2f-46f8bc9de658","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.646412616Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237722023-12-10T18:59:15.664Z{"msg":"af130067-ef73-484b-8814-ffb2d94ba955 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.646495018Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237732023-12-10T18:59:15.664Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.646575681Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237742023-12-10T18:59:15.664Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.646614361Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237752023-12-10T18:59:15.664Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.646674424Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237762023-12-10T18:59:15.665Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.646704274Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237772023-12-10T18:59:15.665Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.646761825Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237782023-12-10T18:59:15.665Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.646785523Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237792023-12-10T18:59:15.665Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
237802023-12-10T18:59:15.665Z,"time":"2023-12-10T18:59:15.646826086Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237812023-12-10T18:59:15.665Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.646931316Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237822023-12-10T18:59:15.665Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.646956932Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237832023-12-10T18:59:15.665Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.646996964Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237842023-12-10T18:59:15.665Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:15.647018647Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1938,"session_id":"ddb8717d-39b9-428c-9a2f-46f8bc9de658"}
237852023-12-10T18:59:15.665ZLoading write count information from file "/tmp/test_repair_verify.data"
237862023-12-10T18:59:15.665ZRun Repair workload
237872023-12-10T18:59:15.665Z01/30 Write block 477 len 32768 data: 10 8 9 9 9 11 12 11
237882023-12-10T18:59:15.665Z02/30 Read block 185 len 8192
237892023-12-10T18:59:15.666Z03/30 Write block 221 len 20480 data: 19 19 17 17 15
237902023-12-10T18:59:15.666Z04/30 Read block 61 len 36864
237912023-12-10T18:59:15.666Z05/30 Write block 516 len 16384 data: 13 12 11 12
237922023-12-10T18:59:15.673Z06/30 Write block 377 len 36864 data: 21 24 22 23 18 17 13 15 17
237932023-12-10T18:59:15.673Z07/30 Read block 582 len 40960
237942023-12-10T18:59:15.673Z08/30 Read block 115 len 4096
237952023-12-10T18:59:15.673Z09/30 Write block 4 len 8192 data: 7 8
237962023-12-10T18:59:15.673Z10/30 Write block 185 len 40960 data: 10 8 7 8 8 11 12 12 12 12
237972023-12-10T18:59:15.673Z11/30 Read block 114 len 8192
237982023-12-10T18:59:15.673Z12/30 Write block 140 len 36864 data: 9 8 9 7 9 9 12 12 14
237992023-12-10T18:59:15.673Z13/30 Flush
238002023-12-10T18:59:15.676Z14/30 Write block 25 len 32768 data: 16 14 10 9 9 12 15 17
238012023-12-10T18:59:15.677Z15/30 Write block 317 len 28672 data: 11 14 16 15 16 16 16
238022023-12-10T18:59:15.679Z16/30 Read block 489 len 20480
238032023-12-10T18:59:15.679Z17/30 Write block 235 len 4096 data: 20
238042023-12-10T18:59:15.682Z18/30 Write block 34 len 4096 data: 13
238052023-12-10T18:59:15.682Z19/30 Read block 185 len 28672
238062023-12-10T18:59:15.685Z20/30 Read block 528 len 24576
238072023-12-10T18:59:15.688Z21/30 Flush
238082023-12-10T18:59:15.718Z22/30 Write block 90 len 4096 data: 16
238092023-12-10T18:59:15.719Z23/30 Write block 161 len 16384 data: 12 19 19 18
238102023-12-10T18:59:15.719Z24/30 Flush
238112023-12-10T18:59:15.722Z25/30 Write block 184 len 12288 data: 11 11 9
238122023-12-10T18:59:15.722Z26/30 Write block 277 len 8192 data: 19 17
238132023-12-10T18:59:15.722Z27/30 Write block 53 len 16384 data: 12 14 13 14
238142023-12-10T18:59:15.723Z28/30 Read block 304 len 40960
238152023-12-10T18:59:15.723Z29/30 Read block 22 len 4096
238162023-12-10T18:59:15.723Z30/30 Write block 509 len 20480 data: 10 9 10 9 10
238172023-12-10T18:59:15.723Z----------------------------------------------------------------
238182023-12-10T18:59:15.723Z Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30
238192023-12-10T18:59:15.723ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
238202023-12-10T18:59:15.724Z 1 Acked 1000 Write 8 Done Sent Done false
238212023-12-10T18:59:15.724Z 2 Acked 1001 Read 2 Done Sent Done false
238222023-12-10T18:59:15.724Z 3 Acked 1002 Write 5 Done Sent Done false
238232023-12-10T18:59:15.724Z 4 Acked 1003 Read 9 Done Sent Done false
238242023-12-10T18:59:15.724Z 5 Acked 1004 Write 4 Done Sent Done false
238252023-12-10T18:59:15.724Z 6 Acked 1005 Write 9 Done Sent Done false
238262023-12-10T18:59:15.724Z 7 Acked 1006 Read 10 Done Sent Done false
238272023-12-10T18:59:15.724Z 8 Acked 1007 Read 1 Done Sent Done false
238282023-12-10T18:59:15.724Z 9 Acked 1008 Write 2 Done Sent Done false
238292023-12-10T18:59:15.724Z 10 Acked 1009 Write 10 Done Sent Done false
238302023-12-10T18:59:15.724Z 11 Acked 1010 Read 2 Done Sent Done false
238312023-12-10T18:59:15.724Z 12 Acked 1011 Write 9 Done Sent Done false
238322023-12-10T18:59:15.724Z 13 Acked 1012 Flush 0 Done Sent Done false
238332023-12-10T18:59:15.724Z 14 Acked 1013 Write 8 Done Sent Done false
238342023-12-10T18:59:15.724Z 15 Acked 1014 Write 7 Done Sent Done false
238352023-12-10T18:59:15.724Z 16 Acked 1015 Read 5 Done Sent Done false
238362023-12-10T18:59:15.724Z 17 Acked 1016 Write 1 Done Sent Done false
238372023-12-10T18:59:15.724Z 18 Acked 1017 Write 1 Done Sent Done false
238382023-12-10T18:59:15.725Z 19 Acked 1018 Read 7 Done Sent Done false
238392023-12-10T18:59:15.725Z 20 Acked 1019 Read 6 Done Sent Done false
238402023-12-10T18:59:15.726Z 21 Acked 1020 Flush 0 Done Sent Done false
238412023-12-10T18:59:15.726Z 22 Acked 1021 Write 1 Done Sent Done false
238422023-12-10T18:59:15.726Z 23 Acked 1022 Write 4 Done Sent Done false
238432023-12-10T18:59:15.726Z 24 Acked 1023 Flush 0 Done Sent Done false
238442023-12-10T18:59:15.726Z 25 Acked 1024 Write 3 Done Sent Done false
238452023-12-10T18:59:15.726Z 26 Acked 1025 Write 2 Done Sent Done false
238462023-12-10T18:59:15.726Z 27 Acked 1026 Write 4 Done Sent Done false
238472023-12-10T18:59:15.726Z 28 Acked 1027 Read 10 Done Sent Done false
238482023-12-10T18:59:15.726Z 29 Acked 1028 Read 1 Done Sent Done false
238492023-12-10T18:59:15.726Z 30 Acked 1029 Write 5 Sent Sent Sent false
238502023-12-10T18:59:15.726Z STATES DS:0 DS:1 DS:2 TOTAL
238512023-12-10T18:59:15.726Z New 0 0 0 0
238522023-12-10T18:59:15.726Z Sent 1 30 1 32
238532023-12-10T18:59:15.726Z Done 29 0 29 58
238542023-12-10T18:59:15.726Z Skipped 0 0 0 0
238552023-12-10T18:59:15.726Z Error 0 0 0 0
238562023-12-10T18:59:15.726ZLast Flush: 1023 0 1023
238572023-12-10T18:59:15.727ZDownstairs last five completed:
238582023-12-10T18:59:15.727ZUpstairs last five completed: 30 29 28 27 26 25
238592023-12-10T18:59:15.727ZWrote out file "/tmp/test_repair_verify.data"
238602023-12-10T18:59:15.727Z
238612023-12-10T18:59:15.727ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
238622023-12-10T18:59:15.737ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
238632023-12-10T18:59:15.737Z 0 000-019 194 192 194 342 337 342 F F F <---
238642023-12-10T18:59:15.741Z 1 020-039 194 191 194 343 336 343 F F F <---
238652023-12-10T18:59:15.741Z 2 040-059 191 191 191 336 336 336 T F T <---
238662023-12-10T18:59:15.741Z 3 060-079 188 188 188 329 329 329 F F F
238672023-12-10T18:59:15.741Z 4 080-099 194 192 194 344 340 344 F F F <---
238682023-12-10T18:59:15.741Z 5 100-119 188 188 188 329 329 329 F F F
238692023-12-10T18:59:15.741Z 6 120-139 189 189 189 332 332 332 F F F
238702023-12-10T18:59:15.741Z 7 140-159 194 192 194 342 340 342 F F F <---
238712023-12-10T18:59:15.741Z 8 160-179 194 192 194 344 340 344 F F F <---
238722023-12-10T18:59:15.741Z 9 180-199 194 187 194 342 327 342 T F T <---
238732023-12-10T18:59:15.741Z 10 200-219 193 193 193 341 341 341 F F F
238742023-12-10T18:59:15.741Z 11 220-239 194 193 194 343 341 343 F F F <---
238752023-12-10T18:59:15.741Z 12 240-259 190 190 190 335 335 335 F F F
238762023-12-10T18:59:15.742Z 13 260-279 188 188 188 329 329 329 T F T <---
238772023-12-10T18:59:15.742Z 14 280-299 188 188 188 329 329 329 F F F
238782023-12-10T18:59:15.742Z 15 300-319 194 191 194 343 336 343 F F F <---
238792023-12-10T18:59:15.742Z 16 320-339 194 190 194 343 333 343 F F F <---
238802023-12-10T18:59:15.742Z 17 340-359 190 190 190 333 333 333 F F F
238812023-12-10T18:59:15.742Z 18 360-379 194 188 194 342 329 342 F F F <---
238822023-12-10T18:59:15.742Z 19 380-399 194 188 194 342 328 342 F F F <---
238832023-12-10T18:59:15.742Z 20 400-419 192 192 192 340 340 340 F F F
238842023-12-10T18:59:15.742Z 21 420-439 192 192 192 340 340 340 F F F
238852023-12-10T18:59:15.742Z 22 440-459 192 192 192 340 340 340 F F F
238862023-12-10T18:59:15.742Z 23 460-479 194 192 194 342 340 342 F F F <---
238872023-12-10T18:59:15.743Z 24 480-499 194 190 194 342 333 342 F F F <---
238882023-12-10T18:59:15.743Z 25 500-519 194 187 194 342 327 342 T F T <---
238892023-12-10T18:59:15.743Z 26 520-539 190 190 190 335 335 335 F F F
238902023-12-10T18:59:15.743Z 27 540-559 188 188 188 329 329 329 F F F
238912023-12-10T18:59:15.743Z 28 560-579 187 187 187 327 327 327 F F F
238922023-12-10T18:59:15.743Z 29 580-599 187 187 187 327 327 327 F F F
238932023-12-10T18:59:15.743ZMax gen: 194, Max flush: 344
238942023-12-10T18:59:15.743ZError: Difference in extent metadata found!
238952023-12-10T18:59:15.747ZA Difference in extent metadata is expected here
238962023-12-10T18:59:15.747ZOn loop 97
238972023-12-10T18:59:15.747Z
238982023-12-10T18:59:15.747ZVerifying data now
238992023-12-10T18:59:21.245ZNo change after verify
239002023-12-10T18:59:21.245ZLoop: 97 Downstairs dump after verify (and repair):
239012023-12-10T18:59:21.268ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
239022023-12-10T18:59:21.268Z 0 000-019 194 194 194 342 342 342 F F F
239032023-12-10T18:59:21.271Z 1 020-039 194 194 194 343 343 343 F F F
239042023-12-10T18:59:21.271Z 2 040-059 195 195 195 345 345 345 F F F
239052023-12-10T18:59:21.271Z 3 060-079 188 188 188 329 329 329 F F F
239062023-12-10T18:59:21.271Z 4 080-099 194 194 194 344 344 344 F F F
239072023-12-10T18:59:21.271Z 5 100-119 188 188 188 329 329 329 F F F
239082023-12-10T18:59:21.271Z 6 120-139 189 189 189 332 332 332 F F F
239092023-12-10T18:59:21.271Z 7 140-159 194 194 194 342 342 342 F F F
239102023-12-10T18:59:21.271Z 8 160-179 194 194 194 344 344 344 F F F
239112023-12-10T18:59:21.272Z 9 180-199 195 195 195 345 345 345 F F F
239122023-12-10T18:59:21.272Z 10 200-219 193 193 193 341 341 341 F F F
239132023-12-10T18:59:21.272Z 11 220-239 194 194 194 343 343 343 F F F
239142023-12-10T18:59:21.272Z 12 240-259 190 190 190 335 335 335 F F F
239152023-12-10T18:59:21.272Z 13 260-279 195 195 195 345 345 345 F F F
239162023-12-10T18:59:21.272Z 14 280-299 188 188 188 329 329 329 F F F
239172023-12-10T18:59:21.272Z 15 300-319 194 194 194 343 343 343 F F F
239182023-12-10T18:59:21.272Z 16 320-339 194 194 194 343 343 343 F F F
239192023-12-10T18:59:21.272Z 17 340-359 190 190 190 333 333 333 F F F
239202023-12-10T18:59:21.272Z 18 360-379 194 194 194 342 342 342 F F F
239212023-12-10T18:59:21.272Z 19 380-399 194 194 194 342 342 342 F F F
239222023-12-10T18:59:21.272Z 20 400-419 192 192 192 340 340 340 F F F
239232023-12-10T18:59:21.272Z 21 420-439 192 192 192 340 340 340 F F F
239242023-12-10T18:59:21.272Z 22 440-459 192 192 192 340 340 340 F F F
239252023-12-10T18:59:21.272Z 23 460-479 194 194 194 342 342 342 F F F
239262023-12-10T18:59:21.272Z 24 480-499 194 194 194 342 342 342 F F F
239272023-12-10T18:59:21.272Z 25 500-519 195 195 195 345 345 345 F F F
239282023-12-10T18:59:21.272Z 26 520-539 190 190 190 335 335 335 F F F
239292023-12-10T18:59:21.273Z 27 540-559 188 188 188 329 329 329 F F F
239302023-12-10T18:59:21.273Z 28 560-579 187 187 187 327 327 327 F F F
239312023-12-10T18:59:21.273Z 29 580-599 187 187 187 327 327 327 F F F
239322023-12-10T18:59:21.273ZMax gen: 195, Max flush: 345
239332023-12-10T18:59:21.276Z
239342023-12-10T18:59:21.276ZBegin loop 98
239352023-12-10T18:59:21.276ZDownstairs to restart: 2
239362023-12-10T18:59:21.305Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.30359039Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946}
239372023-12-10T18:59:21.305Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.304015161Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946}
239382023-12-10T18:59:21.305Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.304044558Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946}
239392023-12-10T18:59:21.310Z{"msg":"Crucible 385f7a90-7e75-4f9c-98c4-eb963d24f9bd has session id: d256290c-1ecd-4970-97d3-b7c30b65fbe1","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.30409112Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239402023-12-10T18:59:21.310Z{"msg":"Upstairs opts: Upstairs UUID: 385f7a90-7e75-4f9c-98c4-eb963d24f9bd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.30419849Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239412023-12-10T18:59:21.310Z{"msg":"Crucible stats registered with UUID: 385f7a90-7e75-4f9c-98c4-eb963d24f9bd","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.30428676Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"Crucible runtime is spawned
239422023-12-10T18:59:21.310Zd256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239432023-12-10T18:59:21.310ZThe guest has requested activation with gen:196
239442023-12-10T18:59:21.310Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.304495798Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"looper":"2","session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239452023-12-10T18:59:21.310Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.304566507Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1","task":"up_listen"}
239462023-12-10T18:59:21.310Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.304600773Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239472023-12-10T18:59:21.310Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.30467085Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239482023-12-10T18:59:21.310Z{"msg":"[2] 385f7a90-7e75-4f9c-98c4-eb963d24f9bd looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.304979951Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"looper":"2","session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239492023-12-10T18:59:21.310Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.305050446Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"looper":"1","session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239502023-12-10T18:59:21.310Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.305248128Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"looper":"0","session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239512023-12-10T18:59:21.310Z{"msg":"[1] 385f7a90-7e75-4f9c-98c4-eb963d24f9bd looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.305332311Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"looper":"1","session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239522023-12-10T18:59:21.310Z{"msg":"[0] 385f7a90-7e75-4f9c-98c4-eb963d24f9bd looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.305492436Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"looper":"0","session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239532023-12-10T18:59:21.311Z{"msg":"385f7a90-7e75-4f9c-98c4-eb963d24f9bd active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.305896957Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239542023-12-10T18:59:21.311Z{"msg":"Set desired generation to :196","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.30595412Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239552023-12-10T18:59:21.311Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.306091871Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239562023-12-10T18:59:21.311Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.306169527Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239572023-12-10T18:59:21.311Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.306607663Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239582023-12-10T18:59:21.311Z{"msg":"[1] 385f7a90-7e75-4f9c-98c4-eb963d24f9bd (d256290c-1ecd-4970-97d3-b7c30b65fbe1) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.306893643Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239592023-12-10T18:59:21.312Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.307043529Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239602023-12-10T18:59:21.312Z{"msg":"[1] client is_active_req TRUE, promote! session d256290c-1ecd-4970-97d3-b7c30b65fbe1","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.307094052Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239612023-12-10T18:59:21.312Z{"msg":"[0] 385f7a90-7e75-4f9c-98c4-eb963d24f9bd (d256290c-1ecd-4970-97d3-b7c30b65fbe1) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.307251167Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239622023-12-10T18:59:21.312Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.307390338Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239632023-12-10T18:59:21.312Z{"msg":"[0] client is_active_req TRUE, promote! session d256290c-1ecd-4970-97d3-b7c30b65fbe1","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.307462839Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239642023-12-10T18:59:21.318Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.307664075Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239652023-12-10T18:59:21.318Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.307795278Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239662023-12-10T18:59:21.318Z{"msg":"385f7a90-7e75-4f9c-98c4-eb963d24f9bd WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.307834283Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239672023-12-10T18:59:21.318Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.30801073Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239682023-12-10T18:59:21.318Z{"msg":"[2] 385f7a90-7e75-4f9c-98c4-eb963d24f9bd (d256290c-1ecd-4970-97d3-b7c30b65fbe1) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.308094863Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239692023-12-10T18:59:21.319Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.30812436Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239702023-12-10T18:59:21.319Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.30819405Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239712023-12-10T18:59:21.319Z{"msg":"[2] client is_active_req TRUE, promote! session d256290c-1ecd-4970-97d3-b7c30b65fbe1","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.308239035Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239722023-12-10T18:59:21.319Z{"msg":"385f7a90-7e75-4f9c-98c4-eb963d24f9bd WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.308279254Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239732023-12-10T18:59:21.319Z{"msg":"[1] 385f7a90-7e75-4f9c-98c4-eb963d24f9bd (d256290c-1ecd-4970-97d3-b7c30b65fbe1) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.308401829Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239742023-12-10T18:59:21.319Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.308464334Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239752023-12-10T18:59:21.319Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:59:21.308551652Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239762023-12-10T18:59:21.319Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.308573334Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239772023-12-10T18:59:21.319Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.308604587Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239782023-12-10T18:59:21.319Z{"msg":"385f7a90-7e75-4f9c-98c4-eb963d24f9bd WaitActive WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.308672562Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239792023-12-10T18:59:21.319Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.308697917Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239802023-12-10T18:59:21.319Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.30873901Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239812023-12-10T18:59:21.319Z{"msg":"[0] 385f7a90-7e75-4f9c-98c4-eb963d24f9bd (d256290c-1ecd-4970-97d3-b7c30b65fbe1) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.308764734Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239822023-12-10T18:59:21.319Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.308785737Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239832023-12-10T18:59:21.319Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:59:21.308805656Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239842023-12-10T18:59:21.319Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.308825862Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239852023-12-10T18:59:21.320Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.308873032Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239862023-12-10T18:59:21.320Z{"msg":"385f7a90-7e75-4f9c-98c4-eb963d24f9bd WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.308899021Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239872023-12-10T18:59:21.320Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.308919839Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239882023-12-10T18:59:21.320Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.308960921Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239892023-12-10T18:59:21.320Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.309134297Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239902023-12-10T18:59:21.320Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.309191245Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239912023-12-10T18:59:21.320Z{"msg":"385f7a90-7e75-4f9c-98c4-eb963d24f9bd WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.309216452Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239922023-12-10T18:59:21.320Z{"msg":"[2] 385f7a90-7e75-4f9c-98c4-eb963d24f9bd (d256290c-1ecd-4970-97d3-b7c30b65fbe1) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.309759379Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239932023-12-10T18:59:21.320Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.309877024Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239942023-12-10T18:59:21.320Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:59:21.309901221Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239952023-12-10T18:59:21.320Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.309921048Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239962023-12-10T18:59:21.320Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.309942724Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239972023-12-10T18:59:21.320Z{"msg":"385f7a90-7e75-4f9c-98c4-eb963d24f9bd WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.30998514Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239982023-12-10T18:59:21.320Z{"msg":"[0]R flush_numbers[0..12]: [342, 343, 345, 329, 344, 329, 332, 342, 344, 345, 341, 343]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.310009327Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
239992023-12-10T18:59:21.320Z{"msg":"[0]R generation[0..12]: [194, 194, 195, 188, 194, 188, 189, 194, 194, 195, 193, 194]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.310051427Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240002023-12-10T18:59:21.320Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.310076255Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240012023-12-10T18:59:21.321Z{"msg":"[1]R flush_numbers[0..12]: [342, 343, 345, 329, 344, 329, 332, 342, 344, 345, 341, 343]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.310122891Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240022023-12-10T18:59:21.321Z{"msg":"[1]R generation[0..12]: [194, 194, 195, 188, 194, 188, 189, 194, 194, 195, 193, 194]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.310146404Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240032023-12-10T18:59:21.321Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:196
240042023-12-10T18:59:21.321ZWait for a query_work_queue command to finish before sending IO
240052023-12-10T18:59:21.321Z,"time":"2023-12-10T18:59:21.31016879Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240062023-12-10T18:59:21.321Z{"msg":"[2]R flush_numbers[0..12]: [342, 343, 345, 329, 344, 329, 332, 342, 344, 345, 341, 343]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.31026939Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240072023-12-10T18:59:21.321Z{"msg":"[2]R generation[0..12]: [194, 194, 195, 188, 194, 188, 189, 194, 194, 195, 193, 194]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.310295506Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240082023-12-10T18:59:21.321Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.310354934Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240092023-12-10T18:59:21.321Z{"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.310398752Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240102023-12-10T18:59:21.322Z{"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.310466872Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240112023-12-10T18:59:21.322Z{"msg":"Next flush: 346","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.310506471Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240122023-12-10T18:59:21.322Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.310575901Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240132023-12-10T18:59:21.322Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.310653353Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240142023-12-10T18:59:21.322Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.310706859Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240152023-12-10T18:59:21.322Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.310748131Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240162023-12-10T18:59:21.322Z{"msg":"385f7a90-7e75-4f9c-98c4-eb963d24f9bd is now active with session: d256290c-1ecd-4970-97d3-b7c30b65fbe1","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.310820859Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240172023-12-10T18:59:21.322Z{"msg":"385f7a90-7e75-4f9c-98c4-eb963d24f9bd Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.310891385Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240182023-12-10T18:59:21.322Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.310934225Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240192023-12-10T18:59:21.322Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.310998994Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240202023-12-10T18:59:21.322Z{"msg":"[2] received reconcile message","v":0,"name":"crucible",Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
240212023-12-10T18:59:21.323Z"level":30,"time":"2023-12-10T18:59:21.311077654Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240222023-12-10T18:59:21.323Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.31114353Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240232023-12-10T18:59:21.323Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.311169477Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240242023-12-10T18:59:21.323Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.311211109Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240252023-12-10T18:59:21.323Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.311233656Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240262023-12-10T18:59:21.323Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.311273631Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240272023-12-10T18:59:21.323Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.311295853Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240282023-12-10T18:59:21.323Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.311336386Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240292023-12-10T18:59:21.324Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:21.311358121Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1946,"session_id":"d256290c-1ecd-4970-97d3-b7c30b65fbe1"}
240302023-12-10T18:59:21.324ZLoading write count information from file "/tmp/test_repair_verify.data"
240312023-12-10T18:59:21.324ZRun Repair workload
240322023-12-10T18:59:21.324Z01/30 Write block 1 len 32768 data: 6 8 8 8 9 8 12 16
240332023-12-10T18:59:21.324Z02/30 Read block 195 len 4096
240342023-12-10T18:59:21.327Z03/30 Write block 521 len 32768 data: 13 14 16 17 17 19 17 17
240352023-12-10T18:59:21.327Z04/30 Write block 575 len 4096 data: 19
240362023-12-10T18:59:21.327Z05/30 Write block 481 len 28672 data: 10 12 13 12 12 13 15
240372023-12-10T18:59:21.327Z06/30 Read block 264 len 4096
240382023-12-10T18:59:21.327Z07/30 Write block 136 len 16384 data: 6 5 8 10
240392023-12-10T18:59:21.328Z08/30 Read block 8 len 36864
240402023-12-10T18:59:21.328Z09/30 Write block 82 len 28672 data: 20 22 22 21 23 21 23
240412023-12-10T18:59:21.328Z10/30 Read block 31 len 20480
240422023-12-10T18:59:21.330Z11/30 Read block 47 len 28672
240432023-12-10T18:59:21.333Z12/30 Read block 498 len 4096
240442023-12-10T18:59:21.333Z13/30 Read block 139 len 4096
240452023-12-10T18:59:21.337Z14/30 Write block 314 len 40960 data: 14 14 12 12 15 17 16 17 17 17
240462023-12-10T18:59:21.337Z15/30 Read block 338 len 8192
240472023-12-10T18:59:21.340Z16/30 Write block 115 len 20480 data: 17 18 17 17 17
240482023-12-10T18:59:21.340Z17/30 Read block 235 len 16384
240492023-12-10T18:59:21.343Z18/30 Flush
240502023-12-10T18:59:21.346Z19/30 Write block 521 len 16384 data: 14 15 17 18
240512023-12-10T18:59:21.348Z20/30 Write block 596 len 12288 data: 7 7 4
240522023-12-10T18:59:21.348Z21/30 Read block 116 len 36864
240532023-12-10T18:59:21.352Z22/30 Read block 127 len 24576
240542023-12-10T18:59:21.354Z23/30 Write block 433 len 20480 data: 13 13 11 9 10
240552023-12-10T18:59:21.354Z24/30 Read block 210 len 40960
240562023-12-10T18:59:21.357Z25/30 Flush
240572023-12-10T18:59:21.360Z26/30 Write block 593 len 16384 data: 10 10 9 8
240582023-12-10T18:59:21.363Z27/30 Write block 478 len 40960 data: 9 10 10 11 13 14 13 13 14 16
240592023-12-10T18:59:21.363Z28/30 Read block 59 len 20480
240602023-12-10T18:59:21.366Z29/30 Read block 304 len 28672
240612023-12-10T18:59:21.373Z30/30 Write block 426 len 16384 data: 17 16 12 13
240622023-12-10T18:59:21.373Z----------------------------------------------------------------
240632023-12-10T18:59:21.374Z Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:30
240642023-12-10T18:59:21.374ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
240652023-12-10T18:59:21.374Z 1 Acked 1000 Write 8 Done Done Done false
240662023-12-10T18:59:21.374Z 2 Acked 1001 Read 1 Done Done Done false
240672023-12-10T18:59:21.374Z 3 Acked 1002 Write 8 Done Done Done false
240682023-12-10T18:59:21.374Z 4 Acked 1003 Write 1 Done Done Done false
240692023-12-10T18:59:21.374Z 5 Acked 1004 Write 7 Done Done Done false
240702023-12-10T18:59:21.374Z 6 Acked 1005 Read 1 Done Done Done false
240712023-12-10T18:59:21.374Z 7 Acked 1006 Write 4 Done Done Done false
240722023-12-10T18:59:21.374Z 8 Acked 1007 Read 9 Done Done Sent false
240732023-12-10T18:59:21.374Z 9 Acked 1008 Write 7 Done Done Sent false
240742023-12-10T18:59:21.374Z 10 Acked 1009 Read 5 Done Done Sent false
240752023-12-10T18:59:21.374Z 11 Acked 1010 Read 7 Done Done Sent false
240762023-12-10T18:59:21.374Z 12 Acked 1011 Read 1 Done Done Sent false
240772023-12-10T18:59:21.374Z 13 Acked 1012 Read 1 Done Done Sent false
240782023-12-10T18:59:21.375Z 14 Acked 1013 Write 10 Done Done Sent false
240792023-12-10T18:59:21.375Z 15 Acked 1014 Read 2 Done Done Sent false
240802023-12-10T18:59:21.375Z 16 Acked 1015 Write 5 Done Done Sent false
240812023-12-10T18:59:21.375Z 17 Acked 1016 Read 4 Done Done Sent false
240822023-12-10T18:59:21.375Z 18 Acked 1017 Flush 0 Done Done Sent false
240832023-12-10T18:59:21.375Z 19 Acked 1018 Write 4 Done Done Sent false
240842023-12-10T18:59:21.375Z 20 Acked 1019 Write 3 Done Done Sent false
240852023-12-10T18:59:21.375Z 21 Acked 1020 Read 9 Done Done Sent false
240862023-12-10T18:59:21.375Z 22 Acked 1021 Read 6 Done Done Sent false
240872023-12-10T18:59:21.375Z 23 Acked 1022 Write 5 Done Done Sent false
240882023-12-10T18:59:21.375Z 24 Acked 1023 Read 10 Done Done Sent false
240892023-12-10T18:59:21.375Z 25 Acked 1024 Flush 0 Done Done Sent false
240902023-12-10T18:59:21.375Z 26 Acked 1025 Write 4 Done Done Sent false
240912023-12-10T18:59:21.375Z 27 Acked 1026 Write 10 Done Done Sent false
240922023-12-10T18:59:21.375Z 28 Acked 1027 Read 5 Done Done Sent false
240932023-12-10T18:59:21.375Z 29 Acked 1028 Read 7 Done Done Sent false
240942023-12-10T18:59:21.375Z 30 Acked 1029 Write 4 Sent Sent Sent false
240952023-12-10T18:59:21.375Z STATES DS:0 DS:1 DS:2 TOTAL
240962023-12-10T18:59:21.376Z New 0 0 0 0
240972023-12-10T18:59:21.376Z Sent 1 1 23 25
240982023-12-10T18:59:21.376Z Done 29 29 7 65
240992023-12-10T18:59:21.376Z Skipped 0 0 0 0
241002023-12-10T18:59:21.376Z Error 0 0 0 0
241012023-12-10T18:59:21.376ZLast Flush: 1024 1024 0
241022023-12-10T18:59:21.376ZDownstairs last five completed:
241032023-12-10T18:59:21.376ZUpstairs last five completed: 30 29 28 27 26 25
241042023-12-10T18:59:21.376ZWrote out file "/tmp/test_repair_verify.data"
241052023-12-10T18:59:21.379Z
241062023-12-10T18:59:21.382ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
241072023-12-10T18:59:21.407ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
241082023-12-10T18:59:21.407Z 0 000-019 196 196 194 346 346 342 F F T <---
241092023-12-10T18:59:21.407Z 1 020-039 194 194 194 343 343 343 F F F
241102023-12-10T18:59:21.411Z 2 040-059 195 195 195 345 345 345 F F F
241112023-12-10T18:59:21.411Z 3 060-079 188 188 188 329 329 329 F F F
241122023-12-10T18:59:21.411Z 4 080-099 196 196 194 346 346 344 F F F <---
241132023-12-10T18:59:21.411Z 5 100-119 196 196 188 346 346 329 F F F <---
241142023-12-10T18:59:21.411Z 6 120-139 196 196 189 346 346 332 F F T <---
241152023-12-10T18:59:21.411Z 7 140-159 194 194 194 342 342 342 F F F
241162023-12-10T18:59:21.411Z 8 160-179 194 194 194 344 344 344 F F F
241172023-12-10T18:59:21.411Z 9 180-199 195 195 195 345 345 345 F F F
241182023-12-10T18:59:21.411Z 10 200-219 193 193 193 341 341 341 F F F
241192023-12-10T18:59:21.411Z 11 220-239 194 194 194 343 343 343 F F F
241202023-12-10T18:59:21.412Z 12 240-259 190 190 190 335 335 335 F F F
241212023-12-10T18:59:21.412Z 13 260-279 195 195 195 345 345 345 F F F
241222023-12-10T18:59:21.412Z 14 280-299 188 188 188 329 329 329 F F F
241232023-12-10T18:59:21.412Z 15 300-319 196 196 194 346 346 343 F F F <---
241242023-12-10T18:59:21.412Z 16 320-339 196 196 194 346 346 343 F F F <---
241252023-12-10T18:59:21.412Z 17 340-359 190 190 190 333 333 333 F F F
241262023-12-10T18:59:21.412Z 18 360-379 194 194 194 342 342 342 F F F
241272023-12-10T18:59:21.412Z 19 380-399 194 194 194 342 342 342 F F F
241282023-12-10T18:59:21.412Z 20 400-419 192 192 192 340 340 340 F F F
241292023-12-10T18:59:21.412Z 21 420-439 196 196 192 347 347 340 T T F <---
241302023-12-10T18:59:21.412Z 22 440-459 192 192 192 340 340 340 F F F
241312023-12-10T18:59:21.412Z 23 460-479 194 194 194 342 342 342 T T F <---
241322023-12-10T18:59:21.412Z 24 480-499 196 196 194 346 346 342 T T T <---
241332023-12-10T18:59:21.412Z 25 500-519 195 195 195 345 345 345 F F F
241342023-12-10T18:59:21.412Z 26 520-539 196 196 190 347 347 335 F F T <---
241352023-12-10T18:59:21.412Z 27 540-559 188 188 188 329 329 329 F F F
241362023-12-10T18:59:21.412Z 28 560-579 196 196 187 346 346 327 F F T <---
241372023-12-10T18:59:21.412Z 29 580-599 196 196 187 347 347 327 T T F <---
241382023-12-10T18:59:21.413ZMax gen: 196, Max flush: 347
241392023-12-10T18:59:21.413ZError: Difference in extent metadata found!
241402023-12-10T18:59:21.416ZA Difference in extent metadata is expected here
241412023-12-10T18:59:21.416ZOn loop 98
241422023-12-10T18:59:21.416Z
241432023-12-10T18:59:21.416ZVerifying data now
241442023-12-10T18:59:27.601ZNo change after verify
241452023-12-10T18:59:27.601ZLoop: 98 Downstairs dump after verify (and repair):
241462023-12-10T18:59:27.624ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
241472023-12-10T18:59:27.624Z 0 000-019 196 196 196 346 346 346 F F F
241482023-12-10T18:59:27.628Z 1 020-039 194 194 194 343 343 343 F F F
241492023-12-10T18:59:27.628Z 2 040-059 195 195 195 345 345 345 F F F
241502023-12-10T18:59:27.628Z 3 060-079 188 188 188 329 329 329 F F F
241512023-12-10T18:59:27.628Z 4 080-099 196 196 196 346 346 346 F F F
241522023-12-10T18:59:27.628Z 5 100-119 196 196 196 346 346 346 F F F
241532023-12-10T18:59:27.628Z 6 120-139 196 196 196 346 346 346 F F F
241542023-12-10T18:59:27.628Z 7 140-159 194 194 194 342 342 342 F F F
241552023-12-10T18:59:27.628Z 8 160-179 194 194 194 344 344 344 F F F
241562023-12-10T18:59:27.628Z 9 180-199 195 195 195 345 345 345 F F F
241572023-12-10T18:59:27.628Z 10 200-219 193 193 193 341 341 341 F F F
241582023-12-10T18:59:27.628Z 11 220-239 194 194 194 343 343 343 F F F
241592023-12-10T18:59:27.628Z 12 240-259 190 190 190 335 335 335 F F F
241602023-12-10T18:59:27.628Z 13 260-279 195 195 195 345 345 345 F F F
241612023-12-10T18:59:27.628Z 14 280-299 188 188 188 329 329 329 F F F
241622023-12-10T18:59:27.628Z 15 300-319 196 196 196 346 346 346 F F F
241632023-12-10T18:59:27.628Z 16 320-339 196 196 196 346 346 346 F F F
241642023-12-10T18:59:27.629Z 17 340-359 190 190 190 333 333 333 F F F
241652023-12-10T18:59:27.629Z 18 360-379 194 194 194 342 342 342 F F F
241662023-12-10T18:59:27.629Z 19 380-399 194 194 194 342 342 342 F F F
241672023-12-10T18:59:27.629Z 20 400-419 192 192 192 340 340 340 F F F
241682023-12-10T18:59:27.629Z 21 420-439 197 197 197 348 348 348 F F F
241692023-12-10T18:59:27.629Z 22 440-459 192 192 192 340 340 340 F F F
241702023-12-10T18:59:27.629Z 23 460-479 197 197 197 348 348 348 F F F
241712023-12-10T18:59:27.629Z 24 480-499 197 197 197 348 348 348 F F F
241722023-12-10T18:59:27.630Z 25 500-519 195 195 195 345 345 345 F F F
241732023-12-10T18:59:27.630Z 26 520-539 196 196 196 347 347 347 F F F
241742023-12-10T18:59:27.630Z 27 540-559 188 188 188 329 329 329 F F F
241752023-12-10T18:59:27.630Z 28 560-579 196 196 196 346 346 346 F F F
241762023-12-10T18:59:27.630Z 29 580-599 197 197 197 348 348 348 F F F
241772023-12-10T18:59:27.630ZMax gen: 197, Max flush: 348
241782023-12-10T18:59:27.633Z
241792023-12-10T18:59:27.633ZBegin loop 99
241802023-12-10T18:59:27.633ZDownstairs to restart: 2
241812023-12-10T18:59:27.659Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.657862523Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954}
241822023-12-10T18:59:27.665Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"2ed025a1b445016a0c73d0c9f3b9d9cefc7e2924\",\n git_commit_timestamp: \"2023-12-10T18:22:02.000000000Z\",\n git_branch: \"renovate/rusqlite-0.x\",\n rustc_semver: \"1.73.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"cc66ad468955717ab92600c770da8c1601a4ff33\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.658283154Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954}
241832023-12-10T18:59:27.665Z{"msg":"Upstairs <-> Downstairs Message Version: 5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.658404212Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954}
241842023-12-10T18:59:27.665ZCrucible runtime is spawned
241852023-12-10T18:59:27.665Z{"msg":"Crucible 42f96eec-6525-467b-a60b-8f25e3bca152 has session id: e2df2d8c-e8b8-4fbb-8a46-0893d12a0960","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.65867651Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
241862023-12-10T18:59:27.665Z{"msg":"The guest has requested activation with gen:198
241872023-12-10T18:59:27.665ZUpstairs opts: Upstairs UUID: 42f96eec-6525-467b-a60b-8f25e3bca152, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.658846722Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
241882023-12-10T18:59:27.665Z{"msg":"Crucible stats registered with UUID: 42f96eec-6525-467b-a60b-8f25e3bca152","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.658923599Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
241892023-12-10T18:59:27.665Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.658988939Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960","task":"up_listen"}
241902023-12-10T18:59:27.665Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.65903366Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
241912023-12-10T18:59:27.665Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.659071668Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"looper":"1","session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
241922023-12-10T18:59:27.665Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.659163916Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
241932023-12-10T18:59:27.665Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.65919388Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"looper":"0","session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
241942023-12-10T18:59:27.666Z{"msg":"[1] 42f96eec-6525-467b-a60b-8f25e3bca152 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.659253633Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"looper":"1","session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
241952023-12-10T18:59:27.666Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.659281189Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"looper":"2","session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
241962023-12-10T18:59:27.666Z{"msg":"[0] 42f96eec-6525-467b-a60b-8f25e3bca152 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.659340754Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"looper":"0","session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
241972023-12-10T18:59:27.666Z{"msg":"[2] 42f96eec-6525-467b-a60b-8f25e3bca152 looper connected","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.659368603Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"looper":"2","session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
241982023-12-10T18:59:27.667Z{"msg":"42f96eec-6525-467b-a60b-8f25e3bca152 active request set","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.659458821Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
241992023-12-10T18:59:27.667Z{"msg":"Set desired generation to :198","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.659520498Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242002023-12-10T18:59:27.667Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.659652894Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242012023-12-10T18:59:27.667Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.659982558Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242022023-12-10T18:59:27.667Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.660087152Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242032023-12-10T18:59:27.667Z{"msg":"[1] 42f96eec-6525-467b-a60b-8f25e3bca152 (e2df2d8c-e8b8-4fbb-8a46-0893d12a0960) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.660438721Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242042023-12-10T18:59:27.667Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.66052276Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242052023-12-10T18:59:27.667Z{"msg":"[1] client is_active_req TRUE, promote! session e2df2d8c-e8b8-4fbb-8a46-0893d12a0960","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.660556089Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242062023-12-10T18:59:27.667Z{"msg":"[0] 42f96eec-6525-467b-a60b-8f25e3bca152 (e2df2d8c-e8b8-4fbb-8a46-0893d12a0960) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.660600838Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242072023-12-10T18:59:27.667Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.660642813Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242082023-12-10T18:59:27.668Z{"msg":"[0] client is_active_req TRUE, promote! session e2df2d8c-e8b8-4fbb-8a46-0893d12a0960","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.660711735Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242092023-12-10T18:59:27.668Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.661064776Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242102023-12-10T18:59:27.668Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.661154285Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242112023-12-10T18:59:27.668Z{"msg":"42f96eec-6525-467b-a60b-8f25e3bca152 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.661198579Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242122023-12-10T18:59:27.668Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.661246742Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242132023-12-10T18:59:27.668Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.661275001Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242142023-12-10T18:59:27.668Z{"msg":"42f96eec-6525-467b-a60b-8f25e3bca152 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.661296545Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242152023-12-10T18:59:27.668Z{"msg":"[2] 42f96eec-6525-467b-a60b-8f25e3bca152 (e2df2d8c-e8b8-4fbb-8a46-0893d12a0960) WaitActive WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.661334345Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242162023-12-10T18:59:27.668Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.661388228Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242172023-12-10T18:59:27.669Z{"msg":"[2] client is_active_req TRUE, promote! session e2df2d8c-e8b8-4fbb-8a46-0893d12a0960","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.661413845Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242182023-12-10T18:59:27.675Z{"msg":"[1] 42f96eec-6525-467b-a60b-8f25e3bca152 (e2df2d8c-e8b8-4fbb-8a46-0893d12a0960) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.661843249Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242192023-12-10T18:59:27.675Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.661924949Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242202023-12-10T18:59:27.675Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:59:27.661968353Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242212023-12-10T18:59:27.675Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.662042696Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242222023-12-10T18:59:27.675Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.66207096Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242232023-12-10T18:59:27.675Z{"msg":"42f96eec-6525-467b-a60b-8f25e3bca152 WaitActive WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.662137138Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242242023-12-10T18:59:27.675Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.662177228Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242252023-12-10T18:59:27.675Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.662245725Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242262023-12-10T18:59:27.675Z{"msg":"[0] 42f96eec-6525-467b-a60b-8f25e3bca152 (e2df2d8c-e8b8-4fbb-8a46-0893d12a0960) WaitActive WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.662291408Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242272023-12-10T18:59:27.677Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.662363194Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242282023-12-10T18:59:27.677Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:59:27.66240579Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242292023-12-10T18:59:27.677Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.662442794Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242302023-12-10T18:59:27.677Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.66248338Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242312023-12-10T18:59:27.677Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.662580075Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242322023-12-10T18:59:27.677Z{"msg":"42f96eec-6525-467b-a60b-8f25e3bca152 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.662618614Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242332023-12-10T18:59:27.677Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.662662462Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242342023-12-10T18:59:27.677Z{"msg":"42f96eec-6525-467b-a60b-8f25e3bca152 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.66268554Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242352023-12-10T18:59:27.677Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.662725567Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242362023-12-10T18:59:27.677Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.662747009Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242372023-12-10T18:59:27.678Z{"msg":"[2] 42f96eec-6525-467b-a60b-8f25e3bca152 (e2df2d8c-e8b8-4fbb-8a46-0893d12a0960) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.663072967Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242382023-12-10T18:59:27.678Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.663136892Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242392023-12-10T18:59:27.678Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-12-10T18:59:27.663162851Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242402023-12-10T18:59:27.678Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.66320434Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242412023-12-10T18:59:27.678Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.663227943Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242422023-12-10T18:59:27.678Z{"msg":"42f96eec-6525-467b-a60b-8f25e3bca152 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.663269571Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242432023-12-10T18:59:27.678Z{"msg":"[0]R flush_numbers[0..12]: [346, 343, 345, 329, 346, 346, 346, 342, 344, 345, 341, 343]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.663294151Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242442023-12-10T18:59:27.679Z{"msg":"[0]R generation[0..12]: [196, 194, 195, 188, 196, 196, 196, 194, 194, 195, 193, 194]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.663350254Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242452023-12-10T18:59:27.679Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0The guest has finished waiting for activation with:198
242462023-12-10T18:59:27.679ZWait for a query_work_queue command to finish before sending IO
242472023-12-10T18:59:27.679Z,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.663532383Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242482023-12-10T18:59:27.679Z{"msg":"[1]R flush_numbers[0..12]: [346, 343, 345, 329, 346, 346, 346, 342, 344, 345, 341, 343]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.663599353Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242492023-12-10T18:59:27.679Z{"msg":"[1]R generation[0..12]: [196, 194, 195, 188, 196, 196, 196, 194, 194, 195, 193, 194]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.663626977Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242502023-12-10T18:59:27.679Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.66364919Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242512023-12-10T18:59:27.679Z{"msg":"[2]R flush_numbers[0..12]: [346, 343, 345, 329, 346, 346, 346, 342, 344, 345, 341, 343]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.663670945Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242522023-12-10T18:59:27.679Z{"msg":"[2]R generation[0..12]: [196, 194, 195, 188, 196, 196, 196, 194, 194, 195, 193, 194]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.663692438Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242532023-12-10T18:59:27.679Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.663753062Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242542023-12-10T18:59:27.679Z{"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.663824096Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242552023-12-10T18:59:27.679Z{"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.663890505Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242562023-12-10T18:59:27.679Z{"msg":"Next flush: 349","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.663916258Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242572023-12-10T18:59:27.679Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.663967748Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242582023-12-10T18:59:27.680Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.663990214Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242592023-12-10T18:59:27.680Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.664011848Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242602023-12-10T18:59:27.680Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.664032486Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242612023-12-10T18:59:27.680Z{"msg":"42f96eec-6525-467b-a60b-8f25e3bca152 is now active with session: e2df2d8c-e8b8-4fbb-8a46-0893d12a0960","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.664054096Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242622023-12-10T18:59:27.680Z{"msg":"42f96eec-6525-467b-a60b-8f25e3bca152 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.66410267Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242632023-12-10T18:59:27.681Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.664125544Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242642023-12-10T18:59:27.681Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.664147052Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242652023-12-10T18:59:27.681Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.664167469Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242662023-12-10T18:59:27.681Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.664188388Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242672023-12-10T18:59:27.681Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.664208176Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242682023-12-10T18:59:27.681Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.664227596Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242692023-12-10T18:59:27.681Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.664276557Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242702023-12-10T18:59:27.681Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.664298275Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242712023-12-10T18:59:27.681Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.664318194Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242722023-12-10T18:59:27.681Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.664360355Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
242732023-12-10T18:59:27.681Z"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242742023-12-10T18:59:27.681Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-12-10T18:59:27.664432034Z","hostname":"ip-10-150-1-104.us-west-2.compute.internal","pid":1954,"session_id":"e2df2d8c-e8b8-4fbb-8a46-0893d12a0960"}
242752023-12-10T18:59:27.681ZLoading write count information from file "/tmp/test_repair_verify.data"
242762023-12-10T18:59:27.681ZRun Repair workload
242772023-12-10T18:59:27.681Z01/30 Write block 64 len 4096 data: 16
242782023-12-10T18:59:27.681Z02/30 Write block 563 len 20480 data: 16 15 15 17 16
242792023-12-10T18:59:27.681Z03/30 Write block 157 len 40960 data: 14 12 14 12 13 20 20 19 18 20
242802023-12-10T18:59:27.682Z04/30 Write block 204 len 28672 data: 18 17 13 13 12 11 11
242812023-12-10T18:59:27.682Z05/30 Write block 397 len 32768 data: 10 11 16 18 17 19 19 16
242822023-12-10T18:59:27.685Z06/30 Write block 543 len 24576 data: 22 20 16 15 15 15
242832023-12-10T18:59:27.685Z07/30 Read block 492 len 32768
242842023-12-10T18:59:27.685Z08/30 Flush
242852023-12-10T18:59:27.685Z09/30 Write block 407 len 16384 data: 17 20 19 17
242862023-12-10T18:59:27.685Z10/30 Read block 54 len 12288
242872023-12-10T18:59:27.685Z11/30 Write block 190 len 24576 data: 12 13 13 13 13 16
242882023-12-10T18:59:27.686Z12/30 Read block 282 len 24576
242892023-12-10T18:59:27.689Z13/30 Flush
242902023-12-10T18:59:27.692Z14/30 Write block 284 len 4096 data: 13
242912023-12-10T18:59:27.692Z15/30 Read block 387 len 36864
242922023-12-10T18:59:27.695Z16/30 Write block 135 len 32768 data: 7 7 6 9 11 10 9 10
242932023-12-10T18:59:27.695Z17/30 Read block 146 len 28672
242942023-12-10T18:59:27.698Z18/30 Write block 259 len 4096 data: 6
242952023-12-10T18:59:27.698Z19/30 Read block 128 len 20480
242962023-12-10T18:59:27.701Z20/30 Flush
242972023-12-10T18:59:27.704Z21/30 Write block 20 len 40960 data: 17 15 14 14 17 17 15 11 10 10
242982023-12-10T18:59:27.704Z22/30 Read block 131 len 4096
242992023-12-10T18:59:27.707Z23/30 Write block 221 len 32768 data: 20 20 18 18 16 19 18 17
243002023-12-10T18:59:27.707Z24/30 Read block 196 len 28672
243012023-12-10T18:59:27.710Z25/30 Read block 586 len 20480
243022023-12-10T18:59:27.713Z26/30 Write block 424 len 4096 data: 19
243032023-12-10T18:59:27.713Z27/30 Write block 60 len 8192 data: 12 13
243042023-12-10T18:59:27.713Z28/30 Write block 16 len 8192 data: 18 20
243052023-12-10T18:59:27.716Z29/30 Write block 440 len 24576 data: 9 10 9 11 11 13
243062023-12-10T18:59:27.716Z30/30 Read block 7 len 28672
243072023-12-10T18:59:27.720Z----------------------------------------------------------------
243082023-12-10T18:59:27.720Z Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30
243092023-12-10T18:59:27.720ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
243102023-12-10T18:59:27.720Z 1 Acked 1000 Write 1 Done Done Done false
243112023-12-10T18:59:27.720Z 2 Acked 1001 Write 5 Done Done Sent false
243122023-12-10T18:59:27.721Z 3 Acked 1002 Write 10 Done Done Sent false
243132023-12-10T18:59:27.721Z 4 Acked 1003 Write 7 Done Done Sent false
243142023-12-10T18:59:27.721Z 5 Acked 1004 Write 8 Done Done Sent false
243152023-12-10T18:59:27.722Z 6 Acked 1005 Write 6 Done Done Sent false
243162023-12-10T18:59:27.722Z 7 Acked 1006 Read 8 Done Done Sent false
243172023-12-10T18:59:27.722Z 8 Acked 1007 Flush 0 Done Done Sent false
243182023-12-10T18:59:27.722Z 9 Acked 1008 Write 4 Done Done Sent false
243192023-12-10T18:59:27.722Z 10 Acked 1009 Read 3 Done Done Sent false
243202023-12-10T18:59:27.722Z 11 Acked 1010 Write 6 Done Done Sent false
243212023-12-10T18:59:27.722Z 12 Acked 1011 Read 6 Done Done Sent false
243222023-12-10T18:59:27.722Z 13 Acked 1012 Flush 0 Done Done Sent false
243232023-12-10T18:59:27.722Z 14 Acked 1013 Write 1 Done Done Sent false
243242023-12-10T18:59:27.722Z 15 Acked 1014 Read 9 Done Done Sent false
243252023-12-10T18:59:27.722Z 16 Acked 1015 Write 8 Done Done Sent false
243262023-12-10T18:59:27.722Z 17 Acked 1016 Read 7 Done Done Sent false
243272023-12-10T18:59:27.722Z 18 Acked 1017 Write 1 Done Done Sent false
243282023-12-10T18:59:27.722Z 19 Acked 1018 Read 5 Done Done Sent false
243292023-12-10T18:59:27.722Z 20 Acked 1019 Flush 0 Done Done Sent false
243302023-12-10T18:59:27.722Z 21 Acked 1020 Write 10 Done Done Sent false
243312023-12-10T18:59:27.722Z 22 Acked 1021 Read 1 Done Done Sent false
243322023-12-10T18:59:27.723Z 23 Acked 1022 Write 8 Done Done Sent false
243332023-12-10T18:59:27.723Z 24 Acked 1023 Read 7 Done Done Sent false
243342023-12-10T18:59:27.723Z 25 Acked 1024 Read 5 Done Done Sent false
243352023-12-10T18:59:27.723Z 26 Acked 1025 Write 1 Done Done Sent false
243362023-12-10T18:59:27.723Z 27 Acked 1026 Write 2 Done Done Sent false
243372023-12-10T18:59:27.723Z 28 Acked 1027 Write 2 Done Done Sent false
243382023-12-10T18:59:27.723Z 29 Acked 1028 Write 6 Done Done Sent false
243392023-12-10T18:59:27.723Z 30 Acked 1029 Read 7 Done Sent Sent false
243402023-12-10T18:59:27.723Z STATES DS:0 DS:1 DS:2 TOTAL
243412023-12-10T18:59:27.723Z New 0 0 0 0
243422023-12-10T18:59:27.723Z Sent 0 1 29 30
243432023-12-10T18:59:27.723Z Done 30 29 1 60
243442023-12-10T18:59:27.723Z Skipped 0 0 0 0
243452023-12-10T18:59:27.723Z Error 0 0 0 0
243462023-12-10T18:59:27.723ZLast Flush: 1019 1019 0
243472023-12-10T18:59:27.723ZDownstairs last five completed:
243482023-12-10T18:59:27.723ZUpstairs last five completed: 30 29 28 27 26 25
243492023-12-10T18:59:27.723ZWrote out file "/tmp/test_repair_verify.data"
243502023-12-10T18:59:27.727Z
243512023-12-10T18:59:27.732ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
243522023-12-10T18:59:27.753ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
243532023-12-10T18:59:27.753Z 0 000-019 196 196 196 346 346 346 T T F <---
243542023-12-10T18:59:27.757Z 1 020-039 194 194 194 343 343 343 T T F <---
243552023-12-10T18:59:27.757Z 2 040-059 195 195 195 345 345 345 F F F
243562023-12-10T18:59:27.757Z 3 060-079 198 198 188 349 349 329 T T T <---
243572023-12-10T18:59:27.757Z 4 080-099 196 196 196 346 346 346 F F F
243582023-12-10T18:59:27.758Z 5 100-119 196 196 196 346 346 346 F F F
243592023-12-10T18:59:27.758Z 6 120-139 198 198 196 351 351 346 F F F <---
243602023-12-10T18:59:27.758Z 7 140-159 198 198 194 351 351 342 F F F <---
243612023-12-10T18:59:27.758Z 8 160-179 198 198 194 349 349 344 F F F <---
243622023-12-10T18:59:27.758Z 9 180-199 198 198 195 350 350 345 F F F <---
243632023-12-10T18:59:27.758Z 10 200-219 198 198 193 349 349 341 F F F <---
243642023-12-10T18:59:27.758Z 11 220-239 194 194 194 343 343 343 T T F <---
243652023-12-10T18:59:27.758Z 12 240-259 198 198 190 351 351 335 F F F <---
243662023-12-10T18:59:27.758Z 13 260-279 195 195 195 345 345 345 F F F
243672023-12-10T18:59:27.758Z 14 280-299 198 198 188 351 351 329 F F F <---
243682023-12-10T18:59:27.758Z 15 300-319 196 196 196 346 346 346 F F F
243692023-12-10T18:59:27.758Z 16 320-339 196 196 196 346 346 346 F F F
243702023-12-10T18:59:27.758Z 17 340-359 190 190 190 333 333 333 F F F
243712023-12-10T18:59:27.759Z 18 360-379 194 194 194 342 342 342 F F F
243722023-12-10T18:59:27.759Z 19 380-399 198 198 194 349 349 342 F F F <---
243732023-12-10T18:59:27.759Z 20 400-419 198 198 192 350 350 340 F F F <---
243742023-12-10T18:59:27.759Z 21 420-439 197 197 197 348 348 348 T T F <---
243752023-12-10T18:59:27.759Z 22 440-459 192 192 192 340 340 340 T T F <---
243762023-12-10T18:59:27.759Z 23 460-479 197 197 197 348 348 348 F F F
243772023-12-10T18:59:27.759Z 24 480-499 197 197 197 348 348 348 F F F
243782023-12-10T18:59:27.759Z 25 500-519 195 195 195 345 345 345 F F F
243792023-12-10T18:59:27.759Z 26 520-539 196 196 196 347 347 347 F F F
243802023-12-10T18:59:27.759Z 27 540-559 198 198 188 349 349 329 F F F <---
243812023-12-10T18:59:27.760Z 28 560-579 198 198 196 349 349 346 F F F <---
243822023-12-10T18:59:27.760Z 29 580-599 197 197 197 348 348 348 F F F
243832023-12-10T18:59:27.760ZMax gen: 198, Max flush: 351
243842023-12-10T18:59:27.760ZError: Difference in extent metadata found!
243852023-12-10T18:59:27.763ZA Difference in extent metadata is expected here
243862023-12-10T18:59:27.763ZOn loop 99
243872023-12-10T18:59:27.763Z
243882023-12-10T18:59:27.763ZVerifying data now
243892023-12-10T18:59:33.320ZNo change after verify
243902023-12-10T18:59:33.320ZLoop: 99 Downstairs dump after verify (and repair):
243912023-12-10T18:59:33.342ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
243922023-12-10T18:59:33.342Z 0 000-019 199 199 199 352 352 352 F F F
243932023-12-10T18:59:33.345Z 1 020-039 199 199 199 352 352 352 F F F
243942023-12-10T18:59:33.345Z 2 040-059 195 195 195 345 345 345 F F F
243952023-12-10T18:59:33.345Z 3 060-079 199 199 199 352 352 352 F F F
243962023-12-10T18:59:33.345Z 4 080-099 196 196 196 346 346 346 F F F
243972023-12-10T18:59:33.345Z 5 100-119 196 196 196 346 346 346 F F F
243982023-12-10T18:59:33.345Z 6 120-139 198 198 198 351 351 351 F F F
243992023-12-10T18:59:33.346Z 7 140-159 198 198 198 351 351 351 F F F
244002023-12-10T18:59:33.346Z 8 160-179 198 198 198 349 349 349 F F F
244012023-12-10T18:59:33.346Z 9 180-199 198 198 198 350 350 350 F F F
244022023-12-10T18:59:33.346Z 10 200-219 198 198 198 349 349 349 F F F
244032023-12-10T18:59:33.347Z 11 220-239 199 199 199 352 352 352 F F F
244042023-12-10T18:59:33.347Z 12 240-259 198 198 198 351 351 351 F F F
244052023-12-10T18:59:33.347Z 13 260-279 195 195 195 345 345 345 F F F
244062023-12-10T18:59:33.347Z 14 280-299 198 198 198 351 351 351 F F F
244072023-12-10T18:59:33.347Z 15 300-319 196 196 196 346 346 346 F F F
244082023-12-10T18:59:33.347Z 16 320-339 196 196 196 346 346 346 F F F
244092023-12-10T18:59:33.347Z 17 340-359 190 190 190 333 333 333 F F F
244102023-12-10T18:59:33.347Z 18 360-379 194 194 194 342 342 342 F F F
244112023-12-10T18:59:33.347Z 19 380-399 198 198 198 349 349 349 F F F
244122023-12-10T18:59:33.347Z 20 400-419 198 198 198 350 350 350 F F F
244132023-12-10T18:59:33.347Z 21 420-439 199 199 199 352 352 352 F F F
244142023-12-10T18:59:33.347Z 22 440-459 199 199 199 352 352 352 F F F
244152023-12-10T18:59:33.347Z 23 460-479 197 197 197 348 348 348 F F F
244162023-12-10T18:59:33.347Z 24 480-499 197 197 197 348 348 348 F F F
244172023-12-10T18:59:33.347Z 25 500-519 195 195 195 345 345 345 F F F
244182023-12-10T18:59:33.347Z 26 520-539 196 196 196 347 347 347 F F F
244192023-12-10T18:59:33.347Z 27 540-559 198 198 198 349 349 349 F F F
244202023-12-10T18:59:33.347Z 28 560-579 198 198 198 349 349 349 F F F
244212023-12-10T18:59:33.348Z 29 580-599 197 197 197 348 348 348 F F F
244222023-12-10T18:59:33.348ZMax gen: 199, Max flush: 352
244232023-12-10T18:59:33.381Z9:47 Test duration
244242023-12-10T18:59:33.381ZTest completed
244252023-12-10T18:59:33.381Z
244262023-12-10T18:59:33.381Zreal 9:45.285457235
244272023-12-10T18:59:33.381Zuser 3:01.262310676
244282023-12-10T18:59:33.381Zsys 26.027159725
244292023-12-10T18:59:33.381Ztrap 0.124611932
244302023-12-10T18:59:33.381Ztflt 0.067735443
244312023-12-10T18:59:33.381Zdflt 0.092984870
244322023-12-10T18:59:33.381Zkflt 0.003599734
244332023-12-10T18:59:33.382Zlock 6:30:57.613782766
244342023-12-10T18:59:33.382Zslp 44:31.573439763
244352023-12-10T18:59:33.382Zlat 8.999410066
244362023-12-10T18:59:33.382Zstop 0.062720126
244372023-12-10T18:59:33.385Zprocess exited: duration 598040 ms, exit code 0
 
244382023-12-10T18:59:33.398Zfound 4 output files
244392023-12-10T18:59:33.398Zuploading: /tmp/test_repair_ds8810.txt (345707 bytes)
244402023-12-10T18:59:34.432Zuploaded: /tmp/test_repair_ds8810.txt
244412023-12-10T18:59:34.432Zuploading: /tmp/test_repair_ds8820.txt (46424 bytes)
244422023-12-10T18:59:35.450Zuploaded: /tmp/test_repair_ds8820.txt
244432023-12-10T18:59:35.450Zuploading: /tmp/test_repair_ds8830.txt (25777 bytes)
244442023-12-10T18:59:36.477Zuploaded: /tmp/test_repair_ds8830.txt
244452023-12-10T18:59:36.477Zuploading: /tmp/test_repair_out.txt (376069 bytes)
244462023-12-10T18:59:37.508Zuploaded: /tmp/test_repair_out.txt