01HAZHJ54P30YXWJXXJZNSEM1F: test-repair

BasicConfig {
    output_rules: [
        "/tmp/*.txt",
        "/tmp/core.*",
    ],
    rust_toolchain: None,
    target: Some(
        "helios-2.0",
    ),
    access_repos: [],
    publish: [],
    skip_clone: true,
}

Buildomat Job: 01HAZHJEX12YK6JC6ZEYA4BN6P

Tags:

Artefacts:

Output:

SEQ GLOBAL TIME DETAILS
12023-09-22T23:21:23.322Zjob dependencies complete; ready to run (waiting for 27 m 46 s)
22023-09-22T23:22:19.487Zjob assigned to worker 01HAZK5FS1MFY2614YFWRJYZ75 (queued for 56 s)
32023-09-22T23:22:20.462Zdownloading input: /input/build/work/bins/crucible-downstairs.gz
42023-09-22T23:22:33.611Zdownloaded input: /input/build/work/bins/crucible-downstairs.gz
52023-09-22T23:22:33.617Zdownloading input: /input/build/work/bins/crucible-hammer.gz
62023-09-22T23:22:46.107Zdownloaded input: /input/build/work/bins/crucible-hammer.gz
72023-09-22T23:22:46.112Zdownloading input: /input/build/work/bins/crutest.gz
82023-09-22T23:23:00.900Zdownloaded input: /input/build/work/bins/crutest.gz
92023-09-22T23:23:00.905Zdownloading input: /input/build/work/bins/dsc.gz
102023-09-22T23:23:09.021Zdownloaded input: /input/build/work/bins/dsc.gz
112023-09-22T23:23:09.024Zdownloading input: /input/build/work/scripts/test_ds.sh
122023-09-22T23:23:09.046Zdownloaded input: /input/build/work/scripts/test_ds.sh
132023-09-22T23:23:09.053Zdownloading input: /input/build/work/scripts/test_live_repair.sh
142023-09-22T23:23:09.069Zdownloaded input: /input/build/work/scripts/test_live_repair.sh
152023-09-22T23:23:09.078Zdownloading input: /input/build/work/scripts/test_repair.sh
162023-09-22T23:23:09.099Zdownloaded input: /input/build/work/scripts/test_repair.sh
172023-09-22T23:23:09.116Zdownloading input: /input/build/work/scripts/test_replay.sh
182023-09-22T23:23:09.124Zdownloaded input: /input/build/work/scripts/test_replay.sh
192023-09-22T23:23:09.135Zdownloading input: /input/build/work/scripts/test_up.sh
202023-09-22T23:23:09.678Zdownloaded input: /input/build/work/scripts/test_up.sh
212023-09-22T23:23:09.682Zdownloading input: /input/build/tmp/cargo-test-out.log
222023-09-22T23:23:09.992Zdownloaded input: /input/build/tmp/cargo-test-out.log
 
232023-09-22T23:23:09.997Zstarting task 0: "setup"
242023-09-22T23:23:10.302Z++ uname -s
252023-09-22T23:23:10.307Z+ kern=SunOS
262023-09-22T23:23:10.310Z+ case "$kern" in
272023-09-22T23:23:10.313Z+ groupadd -g 12345 build
282023-09-22T23:23:10.316Z+ useradd -u 12345 -g build -d /home/build -s /bin/bash -c build -P 'Primary Administrator' build
292023-09-22T23:23:12.324Z+ zfs create -o mountpoint=/work rpool/work
302023-09-22T23:23:12.722Z++ awk '$2 == "/home" { print $3 }' /etc/mnttab
312023-09-22T23:23:12.725Z+ home_fs=zfs
322023-09-22T23:23:12.728Z+ [[ zfs == autofs ]]
332023-09-22T23:23:12.731Z+ mkdir -p /home/build
342023-09-22T23:23:12.734Z+ chown build:build /home/build /work
352023-09-22T23:23:14.731Z+ chmod 0700 /home/build /work
362023-09-22T23:23:14.734Zprocess exited: duration 4733 ms, exit code 0
 
372023-09-22T23:23:14.747Zstarting task 1: "authentication"
382023-09-22T23:23:14.777Zprocess exited: duration 24 ms, exit code 0
 
392023-09-22T23:23:14.784Zstarting task 2: "build"
402023-09-22T23:23:14.793Z+ banner cores
412023-09-22T23:23:14.928Z
422023-09-22T23:23:14.931Z #### #### ##### ###### ####
432023-09-22T23:23:14.934Z # # # # # # # #
442023-09-22T23:23:14.937Z # # # # # ##### ####
452023-09-22T23:23:14.940Z # # # ##### # #
462023-09-22T23:23:14.943Z # # # # # # # # #
472023-09-22T23:23:14.946Z #### #### # # ###### ####
482023-09-22T23:23:14.949Z
492023-09-22T23:23:14.952Z+ pfexec coreadm -i /tmp/core.%f.%p -g /tmp/core.%f.%p -e global -e log -e proc-setid -e global-setid
502023-09-22T23:23:14.955Z+ echo 'input bins dir contains:'
512023-09-22T23:23:14.957Zinput bins dir contains:
522023-09-22T23:23:14.960Z+ ls -ltr /input/build/work/bins
532023-09-22T23:23:14.963Ztotal 703419
542023-09-22T23:23:14.966Z-rw-r--r-- 1 root root 108909782 Sep 22 23:22 crucible-downstairs.gz
552023-09-22T23:23:14.968Z-rw-r--r-- 1 root root 86315868 Sep 22 23:22 crucible-hammer.gz
562023-09-22T23:23:14.971Z-rw-r--r-- 1 root root 96159197 Sep 22 23:22 crutest.gz
572023-09-22T23:23:14.974Z-rw-r--r-- 1 root root 68523274 Sep 22 23:23 dsc.gz
582023-09-22T23:23:14.977Z+ echo 'input script dir contains:'
592023-09-22T23:23:14.980Zinput script dir contains:
602023-09-22T23:23:14.982Z+ ls -ltr /input/build/work/scripts
612023-09-22T23:23:14.985Ztotal 32
622023-09-22T23:23:14.988Z-rw-r--r-- 1 root root 939 Sep 22 23:23 test_ds.sh
632023-09-22T23:23:14.991Z-rw-r--r-- 1 root root 3746 Sep 22 23:23 test_live_repair.sh
642023-09-22T23:23:14.994Z-rw-r--r-- 1 root root 6416 Sep 22 23:23 test_repair.sh
652023-09-22T23:23:14.997Z-rw-r--r-- 1 root root 2969 Sep 22 23:23 test_replay.sh
662023-09-22T23:23:14.999Z-rw-r--r-- 1 root root 9311 Sep 22 23:23 test_up.sh
672023-09-22T23:23:15.002Z+ pfexec chmod +x /input/build/work/scripts/test_ds.sh /input/build/work/scripts/test_live_repair.sh /input/build/work/scripts/test_repair.sh /input/build/work/scripts/test_replay.sh /input/build/work/scripts/test_up.sh
682023-09-22T23:23:15.005Z+ echo ' chmod input script dir contains:'
692023-09-22T23:23:15.008Z chmod input script dir contains:
702023-09-22T23:23:15.011Z+ ls -ltr /input/build/work/scripts
712023-09-22T23:23:15.014Ztotal 32
722023-09-22T23:23:15.017Z-rwxr-xr-x 1 root root 939 Sep 22 23:23 test_ds.sh
732023-09-22T23:23:15.020Z-rwxr-xr-x 1 root root 3746 Sep 22 23:23 test_live_repair.sh
742023-09-22T23:23:15.023Z-rwxr-xr-x 1 root root 6416 Sep 22 23:23 test_repair.sh
752023-09-22T23:23:15.026Z-rwxr-xr-x 1 root root 2969 Sep 22 23:23 test_replay.sh
762023-09-22T23:23:15.029Z-rwxr-xr-x 1 root root 9311 Sep 22 23:23 test_up.sh
772023-09-22T23:23:15.031Z+ banner unpack
782023-09-22T23:23:15.034Z
792023-09-22T23:23:15.037Z # # # # ##### ## #### # #
802023-09-22T23:23:15.040Z # # ## # # # # # # # # #
812023-09-22T23:23:15.043Z # # # # # # # # # # ####
822023-09-22T23:23:15.046Z # # # # # ##### ###### # # #
832023-09-22T23:23:15.049Z # # # ## # # # # # # #
842023-09-22T23:23:15.051Z #### # # # # # #### # #
852023-09-22T23:23:15.054Z
862023-09-22T23:23:15.057Z+ mkdir -p /var/tmp/bins
872023-09-22T23:23:15.060Z+ for t in "$input/bins/"*.gz
882023-09-22T23:23:15.066Z++ basename /input/build/work/bins/crucible-downstairs.gz
892023-09-22T23:23:15.076Z+ b=crucible-downstairs.gz
902023-09-22T23:23:15.084Z+ b=crucible-downstairs
912023-09-22T23:23:15.095Z+ gunzip
922023-09-22T23:23:18.806Z+ chmod +x /var/tmp/bins/crucible-downstairs
932023-09-22T23:23:18.811Z+ for t in "$input/bins/"*.gz
942023-09-22T23:23:18.819Z++ basename /input/build/work/bins/crucible-hammer.gz
952023-09-22T23:23:18.823Z+ b=crucible-hammer.gz
962023-09-22T23:23:18.830Z+ b=crucible-hammer
972023-09-22T23:23:18.868Z+ gunzip
982023-09-22T23:23:21.763Z+ chmod +x /var/tmp/bins/crucible-hammer
992023-09-22T23:23:21.766Z+ for t in "$input/bins/"*.gz
1002023-09-22T23:23:21.769Z++ basename /input/build/work/bins/crutest.gz
1012023-09-22T23:23:21.772Z+ b=crutest.gz
1022023-09-22T23:23:21.775Z+ b=crutest
1032023-09-22T23:23:21.778Z+ gunzip
1042023-09-22T23:23:25.050Z+ chmod +x /var/tmp/bins/crutest
1052023-09-22T23:23:25.053Z+ for t in "$input/bins/"*.gz
1062023-09-22T23:23:25.056Z++ basename /input/build/work/bins/dsc.gz
1072023-09-22T23:23:25.059Z+ b=dsc.gz
1082023-09-22T23:23:25.062Z+ b=dsc
1092023-09-22T23:23:25.065Z+ gunzip
1102023-09-22T23:23:27.408Z+ chmod +x /var/tmp/bins/dsc
1112023-09-22T23:23:27.411Z+ export BINDIR=/var/tmp/bins
1122023-09-22T23:23:27.415Z+ BINDIR=/var/tmp/bins
1132023-09-22T23:23:27.417Z+ banner repair
1142023-09-22T23:23:27.421Z
1152023-09-22T23:23:27.425Z ##### ###### ##### ## # #####
1162023-09-22T23:23:27.430Z # # # # # # # # # #
1172023-09-22T23:23:27.434Z # # ##### # # # # # # #
1182023-09-22T23:23:27.444Z ##### # ##### ###### # #####
1192023-09-22T23:23:27.449Z # # # # # # # # #
1202023-09-22T23:23:27.455Z # # ###### # # # # # #
1212023-09-22T23:23:27.473Z
1222023-09-22T23:23:27.496Z+ ptime -m bash /input/build/work/scripts/test_repair.sh -N
1232023-09-22T23:23:27.506ZTurn off color for downstairs dump
1242023-09-22T23:23:27.517ZCreating region directory at: /var/tmp/test_repair
1252023-09-22T23:23:27.530ZCreating dsc directory at: /tmp/dsc
1262023-09-22T23:23:28.086ZDownstairs region 0 created at /var/tmp/test_repair/8810 in 0.63161236
1272023-09-22T23:23:28.150ZDownstairs region 1 created at /var/tmp/test_repair/8820 in 0.06431539
1282023-09-22T23:23:28.213ZDownstairs region 2 created at /var/tmp/test_repair/8830 in 0.06382795
1292023-09-22T23:23:28.219ZRegion set with 3 regions was created
1302023-09-22T23:23:28.232Z/var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1158 1159 1160
1312023-09-22T23:23:28.235ZDump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
1322023-09-22T23:23:28.286Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.181327695Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1332023-09-22T23:23:28.289Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.182772217Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1342023-09-22T23:23:28.292Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.182807142Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1352023-09-22T23:23:28.295Z{"msg":"Crucible stats registered with UUID: 35df58f4-63aa-42d9-aa43-ca413ecb2a84","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.182871188Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1362023-09-22T23:23:28.298Z{"msg":"Crucible 35df58f4-63aa-42d9-aa43-ca413ecb2a84 has session id: 8ac8722c-4f7c-4168-bf0d-7467b341bed2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.182939374Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1372023-09-22T23:23:28.301ZCrucible runtime is spawned
1382023-09-22T23:23:28.304Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30The guest has requested activation with gen:1
1392023-09-22T23:23:28.307Z,"time":"2023-09-22T23:23:27.184877854Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162,"task":"up_listen"}
1402023-09-22T23:23:28.310Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.1850646Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1412023-09-22T23:23:28.312Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.185182715Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1422023-09-22T23:23:28.315Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.185550967Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162,"looper":"0"}
1432023-09-22T23:23:28.318Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.185720777Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162,"looper":"2"}
1442023-09-22T23:23:28.321Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.185817631Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162,"looper":"1"}
1452023-09-22T23:23:28.324Z{"msg":"35df58f4-63aa-42d9-aa43-ca413ecb2a84 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.186074548Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1462023-09-22T23:23:28.327Z{"msg":"Set desired generation to :1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.186120686Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1472023-09-22T23:23:28.330Z{"msg":"[0] 35df58f4-63aa-42d9-aa43-ca413ecb2a84 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.186486145Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162,"looper":"0"}
1482023-09-22T23:23:28.334Z{"msg":"[1] 35df58f4-63aa-42d9-aa43-ca413ecb2a84 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.186634668Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162,"looper":"1"}
1492023-09-22T23:23:28.337Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.186894944Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1502023-09-22T23:23:28.340Z{"msg":"[2] 35df58f4-63aa-42d9-aa43-ca413ecb2a84 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.187145375Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162,"looper":"2"}
1512023-09-22T23:23:28.343Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.187291853Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1522023-09-22T23:23:28.346Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.187948784Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1532023-09-22T23:23:28.349Z{"msg":"[2] 35df58f4-63aa-42d9-aa43-ca413ecb2a84 (6ce02f44-29a1-45fb-bb63-a963a64a65c6) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.191321096Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1542023-09-22T23:23:28.352Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.191379235Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1552023-09-22T23:23:28.355Z{"msg":"[0] 35df58f4-63aa-42d9-aa43-ca413ecb2a84 (6ce02f44-29a1-45fb-bb63-a963a64a65c6) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.191488179Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1562023-09-22T23:23:28.358Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.191527154Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1572023-09-22T23:23:28.361Z{"msg":"[1] 35df58f4-63aa-42d9-aa43-ca413ecb2a84 (6ce02f44-29a1-45fb-bb63-a963a64a65c6) WaitActive New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.191599167Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1582023-09-22T23:23:28.364Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.191667349Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1592023-09-22T23:23:28.367Z{"msg":"[2] client is_active_req TRUE, promote! session 6ce02f44-29a1-45fb-bb63-a963a64a65c6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.191734971Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1602023-09-22T23:23:28.370Z{"msg":"[0] client is_active_req TRUE, promote! session 6ce02f44-29a1-45fb-bb63-a963a64a65c6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.191793608Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1612023-09-22T23:23:28.373Z{"msg":"[1] client is_active_req TRUE, promote! session 6ce02f44-29a1-45fb-bb63-a963a64a65c6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.191837708Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1622023-09-22T23:23:28.375Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.194473705Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1632023-09-22T23:23:28.378Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.194552707Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1642023-09-22T23:23:28.381Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.19460649Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1652023-09-22T23:23:28.384Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.194674911Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1662023-09-22T23:23:28.387Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.194747501Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1672023-09-22T23:23:28.389Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.194828678Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1682023-09-22T23:23:28.392Z{"msg":"35df58f4-63aa-42d9-aa43-ca413ecb2a84 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.195165568Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1692023-09-22T23:23:28.395Z{"msg":"35df58f4-63aa-42d9-aa43-ca413ecb2a84 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.195227466Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1702023-09-22T23:23:28.398Z{"msg":"35df58f4-63aa-42d9-aa43-ca413ecb2a84 WaitActive WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.195358306Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1712023-09-22T23:23:28.401Z{"msg":"[2] 35df58f4-63aa-42d9-aa43-ca413ecb2a84 (6ce02f44-29a1-45fb-bb63-a963a64a65c6) WaitActive WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.197818619Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1722023-09-22T23:23:28.404Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.197903348Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1732023-09-22T23:23:28.407Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:23:27.198112901Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1742023-09-22T23:23:28.410Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.198325972Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1752023-09-22T23:23:28.413Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.198441266Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1762023-09-22T23:23:28.416Z{"msg":"35df58f4-63aa-42d9-aa43-ca413ecb2a84 WaitActive WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.198484827Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1772023-09-22T23:23:28.419Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.19855398Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1782023-09-22T23:23:28.421Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.198616428Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1792023-09-22T23:23:28.424Z{"msg":"[1] 35df58f4-63aa-42d9-aa43-ca413ecb2a84 (6ce02f44-29a1-45fb-bb63-a963a64a65c6) WaitActive WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.198688098Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1802023-09-22T23:23:28.427Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.19875621Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1812023-09-22T23:23:28.430Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:23:27.198825866Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1822023-09-22T23:23:28.433Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.19889633Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1832023-09-22T23:23:28.436Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.198968599Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1842023-09-22T23:23:28.439Z{"msg":"35df58f4-63aa-42d9-aa43-ca413ecb2a84 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.199061789Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1852023-09-22T23:23:28.441Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.199129336Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1862023-09-22T23:23:28.444Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.199194077Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1872023-09-22T23:23:28.447Z{"msg":"[0] 35df58f4-63aa-42d9-aa43-ca413ecb2a84 (6ce02f44-29a1-45fb-bb63-a963a64a65c6) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.199265137Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1882023-09-22T23:23:28.450Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.199337609Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1892023-09-22T23:23:28.452Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:23:27.199405178Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1902023-09-22T23:23:28.455Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.199482411Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1912023-09-22T23:23:28.458Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.199548735Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1922023-09-22T23:23:28.466Z{"msg":"35df58f4-63aa-42d9-aa43-ca413ecb2a84 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.199616228Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1932023-09-22T23:23:28.469Z{"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.199684728Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1942023-09-22T23:23:28.888Z{"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.199755022Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1952023-09-22T23:23:28.892Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.199832574Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1962023-09-22T23:23:28.898Z{"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.19990204Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1972023-09-22T23:23:28.906Z{"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.199970453Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1982023-09-22T23:23:28.913Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.200067452Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
1992023-09-22T23:23:28.924Z{"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.200138891Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
2002023-09-22T23:23:28.932Z{"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.200205297Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
2012023-09-22T23:23:28.944Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.200275251Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
2022023-09-22T23:23:28.950Z{"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.200342871Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
2032023-09-22T23:23:28.955Z{"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.200411568Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
2042023-09-22T23:23:28.960Z{"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.20044852Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
2052023-09-22T23:23:28.964Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.200483209Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
2062023-09-22T23:23:28.969Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.200516917Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
2072023-09-22T23:23:28.973Z{The guest has finished waiting for activation with:1
2082023-09-22T23:23:28.976Z"msg":"Wait for a query_work_queue command to finish before sending IO
2092023-09-22T23:23:28.982ZNo initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.200570055Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
2102023-09-22T23:23:28.984Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.200609345Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
2112023-09-22T23:23:28.987Z{"msg":"35df58f4-63aa-42d9-aa43-ca413ecb2a84 is now active with session: 6ce02f44-29a1-45fb-bb63-a963a64a65c6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.200645727Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
2122023-09-22T23:23:28.990Z{"msg":"35df58f4-63aa-42d9-aa43-ca413ecb2a84 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.200680296Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
2132023-09-22T23:23:28.993Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.200714655Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
2142023-09-22T23:23:28.996Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.200741826Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
2152023-09-22T23:23:28.999Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.200772595Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
2162023-09-22T23:23:29.002Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.200809008Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
2172023-09-22T23:23:29.005Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.200841929Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
2182023-09-22T23:23:29.007Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.200876966Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
2192023-09-22T23:23:29.010Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.200908179Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
2202023-09-22T23:23:29.014Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.200927897Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
2212023-09-22T23:23:29.017Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.200945209Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
2222023-09-22T23:23:29.019Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.200962788Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
2232023-09-22T23:23:29.022Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:27.20101318Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
2242023-09-22T23:23:29.026ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
2252023-09-22T23:23:29.029ZFill test
2262023-09-22T23:23:29.306Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:28.202392Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
2272023-09-22T23:23:29.309Z{"msg":"No Live Repair required at this time","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:28.202484021Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1162}
2282023-09-22T23:23:30.597ZRead and Verify all blocks (0..600 range:false)
2292023-09-22T23:23:30.872ZWrote out file "/tmp/test_repair_verify.data"
2302023-09-22T23:23:30.876ZCLIENT: Tests done. All submitted work has been ACK'd
2312023-09-22T23:23:30.879Z----------------------------------------------------------------
2322023-09-22T23:23:30.882Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:1
2332023-09-22T23:23:30.885ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
2342023-09-22T23:23:30.887Z 14 Acked 1013 Read 100 Done Done Done false
2352023-09-22T23:23:30.890Z STATES DS:0 DS:1 DS:2 TOTAL
2362023-09-22T23:23:30.893Z New 0 0 0 0
2372023-09-22T23:23:30.896Z Sent 0 0 0 0
2382023-09-22T23:23:30.901Z Done 1 1 1 3
2392023-09-22T23:23:30.905Z Skipped 0 0 0 0
2402023-09-22T23:23:30.908Z Error 0 0 0 0
2412023-09-22T23:23:30.911ZLast Flush: 1012 1012 1012
2422023-09-22T23:23:30.914ZDownstairs last five completed: 1012 1011 1010 1009 1008 1007
2432023-09-22T23:23:30.917ZUpstairs last five completed: 14 13 12 11 10 9
2442023-09-22T23:23:30.920ZCLIENT: Up:0 ds:1 act:3
2452023-09-22T23:23:34.863Z----------------------------------------------------------------
2462023-09-22T23:23:34.869Z Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0
2472023-09-22T23:23:34.883ZDownstairs last five completed: 1014 1013 1012 1011 1010 1009
2482023-09-22T23:23:34.890ZUpstairs last five completed: 15 14 13 12 11 10
2492023-09-22T23:23:34.907ZCLIENT: Up:0 ds:0 act:3
2502023-09-22T23:23:34.916ZCLIENT: All crucible jobs finished, exiting program
2512023-09-22T23:23:35.338Z
2522023-09-22T23:23:35.341ZBegin loop 1
2532023-09-22T23:23:35.345ZDownstairs to restart: 2
2542023-09-22T23:23:35.348Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.794726591Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2552023-09-22T23:23:35.351Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.79510101Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2562023-09-22T23:23:35.354Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.79515796Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2572023-09-22T23:23:35.357Z{"msg":"Crucible stats registered with UUID: de66e220-9f09-42dc-9032-296fb2dad3da","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.795195906Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2582023-09-22T23:23:35.359Z{"msg":"Crucible de66e220-9f09-42dc-9032-296fb2dad3da has session id: afc4e0dd-9522-460e-a8d9-13895b8d716a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.795262416Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2592023-09-22T23:23:35.362ZCrucible runtime is spawned
2602023-09-22T23:23:35.365ZThe guest has requested activation with gen:2
2612023-09-22T23:23:35.368Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.795562211Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164,"task":"up_listen"}
2622023-09-22T23:23:35.371Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.795618652Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2632023-09-22T23:23:35.374Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.795658914Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2642023-09-22T23:23:35.376Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.795698573Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164,"looper":"1"}
2652023-09-22T23:23:35.379Z{"msg":"[1] de66e220-9f09-42dc-9032-296fb2dad3da looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.795944161Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164,"looper":"1"}
2662023-09-22T23:23:35.382Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.796025877Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164,"looper":"0"}
2672023-09-22T23:23:35.385Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.796227804Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164,"looper":"2"}
2682023-09-22T23:23:35.388Z{"msg":"[0] de66e220-9f09-42dc-9032-296fb2dad3da looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.796298579Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164,"looper":"0"}
2692023-09-22T23:23:35.391Z{"msg":"de66e220-9f09-42dc-9032-296fb2dad3da active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.796704252Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2702023-09-22T23:23:35.394Z{"msg":"Set desired generation to :2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.796757752Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2712023-09-22T23:23:35.396Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.796798998Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2722023-09-22T23:23:35.399Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.797173337Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2732023-09-22T23:23:35.402Z{"msg":"[1] de66e220-9f09-42dc-9032-296fb2dad3da (0f97b488-efb9-4d41-93dd-a786baa39094) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.797504809Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2742023-09-22T23:23:35.405Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.797544029Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2752023-09-22T23:23:35.407Z{"msg":"[1] client is_active_req TRUE, promote! session 0f97b488-efb9-4d41-93dd-a786baa39094","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.797566548Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2762023-09-22T23:23:35.410Z{"msg":"[0] de66e220-9f09-42dc-9032-296fb2dad3da (0f97b488-efb9-4d41-93dd-a786baa39094) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.797712724Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2772023-09-22T23:23:35.414Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.797778364Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2782023-09-22T23:23:35.417Z{"msg":"[0] client is_active_req TRUE, promote! session 0f97b488-efb9-4d41-93dd-a786baa39094","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.797850862Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2792023-09-22T23:23:35.420Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.798195928Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2802023-09-22T23:23:35.423Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.798258444Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2812023-09-22T23:23:35.434Z{"msg":"de66e220-9f09-42dc-9032-296fb2dad3da WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.798349099Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2822023-09-22T23:23:35.448Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.798413383Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2832023-09-22T23:23:35.855Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.798462997Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2842023-09-22T23:23:35.858Z{"msg":"de66e220-9f09-42dc-9032-296fb2dad3da WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.798504023Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2852023-09-22T23:23:35.865Z{"msg":"[1] de66e220-9f09-42dc-9032-296fb2dad3da (0f97b488-efb9-4d41-93dd-a786baa39094) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.799903498Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2862023-09-22T23:23:35.868Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.799964839Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2872023-09-22T23:23:35.870Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:23:33.799989815Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2882023-09-22T23:23:35.873Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.800043094Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2892023-09-22T23:23:35.876Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.800067796Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2902023-09-22T23:23:35.878Z{"msg":"de66e220-9f09-42dc-9032-296fb2dad3da WaitActive WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.800106013Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2912023-09-22T23:23:35.881Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.800126277Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2922023-09-22T23:23:35.884Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.800175443Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2932023-09-22T23:23:35.890Z{"msg":"[0] de66e220-9f09-42dc-9032-296fb2dad3da (0f97b488-efb9-4d41-93dd-a786baa39094) WaitActive WaitQuorum New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.800980749Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2942023-09-22T23:23:35.899Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.801042911Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2952023-09-22T23:23:35.911Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:23:33.801065944Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2962023-09-22T23:23:35.923Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.801083663Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2972023-09-22T23:23:35.929Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.801121051Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2982023-09-22T23:23:35.935Z{"msg":"de66e220-9f09-42dc-9032-296fb2dad3da WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.801160275Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
2992023-09-22T23:23:35.940Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.801198786Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3002023-09-22T23:23:35.944Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:33.80123645Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3012023-09-22T23:23:35.948Z{"msg":"[2] de66e220-9f09-42dc-9032-296fb2dad3da looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.798295334Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164,"looper":"2"}
3022023-09-22T23:23:35.951Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.798378815Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3032023-09-22T23:23:35.957Z{"msg":"[2] de66e220-9f09-42dc-9032-296fb2dad3da (0f97b488-efb9-4d41-93dd-a786baa39094) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.799473624Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3042023-09-22T23:23:35.960Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.799533527Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3052023-09-22T23:23:35.963Z{"msg":"[2] client is_active_req TRUE, promote! session 0f97b488-efb9-4d41-93dd-a786baa39094","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.799577938Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3062023-09-22T23:23:35.966Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.80011235Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3072023-09-22T23:23:35.969Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.800172521Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3082023-09-22T23:23:35.972Z{"msg":"de66e220-9f09-42dc-9032-296fb2dad3da WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.800215697Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3092023-09-22T23:23:35.975Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.801360437Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3102023-09-22T23:23:35.977Z{"msg":"[2] de66e220-9f09-42dc-9032-296fb2dad3da (0f97b488-efb9-4d41-93dd-a786baa39094) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.802380761Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3112023-09-22T23:23:35.981Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.802510212Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3122023-09-22T23:23:36.004Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:23:34.802581408Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3132023-09-22T23:23:36.007Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.802626121Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3142023-09-22T23:23:36.010Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.802666684Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3152023-09-22T23:23:36.013Z{"msg":"de66e220-9f09-42dc-9032-296fb2dad3da WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.802706637Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3162023-09-22T23:23:36.016Z{"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.802760302Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3172023-09-22T23:23:36.018Z{"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.802781543Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3182023-09-22T23:23:36.021Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.802801411Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3192023-09-22T23:23:36.024Z{"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.802873441Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3202023-09-22T23:23:36.027Z{"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.802909427Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3212023-09-22T23:23:36.030ZThe guest has finished waiting for activation with:2
3222023-09-22T23:23:36.033Z{"msg":"Wait for a query_work_queue command to finish before sending IO
3232023-09-22T23:23:36.035Z[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.80297064Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3242023-09-22T23:23:36.038Z{"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.802994983Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3252023-09-22T23:23:36.041Z{"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.803038247Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3262023-09-22T23:23:36.044Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.803060065Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3272023-09-22T23:23:36.046Z{"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.803097Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3282023-09-22T23:23:36.049Z{"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.803140239Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3292023-09-22T23:23:36.052Z{"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.803178973Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3302023-09-22T23:23:36.056Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.803216688Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3312023-09-22T23:23:36.059Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.803258366Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3322023-09-22T23:23:36.062Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.803310297Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3332023-09-22T23:23:36.065Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.803348918Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3342023-09-22T23:23:36.068Z{"msg":"de66e220-9f09-42dc-9032-296fb2dad3da is now active with session: 0f97b488-efb9-4d41-93dd-a786baa39094","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.803389022Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3352023-09-22T23:23:36.075Z{"msg":"de66e220-9f09-42dc-9032-296fb2dad3da Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.803409726Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3362023-09-22T23:23:36.078Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.80342801Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3372023-09-22T23:23:36.081Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.803469375Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3382023-09-22T23:23:36.084Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.803489091Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3392023-09-22T23:23:36.087Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.803534412Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3402023-09-22T23:23:36.090Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.803578701Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3412023-09-22T23:23:36.093Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.803638929Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3422023-09-22T23:23:36.095Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.803659701Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3432023-09-22T23:23:36.099Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.803705275Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3442023-09-22T23:23:36.102Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.803727391Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3452023-09-22T23:23:36.104Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.803776457Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3462023-09-22T23:23:36.107Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.803797997Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1164}
3472023-09-22T23:23:36.110ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
3482023-09-22T23:23:36.113ZLoading write count information from file "/tmp/test_repair_verify.data"
3492023-09-22T23:23:36.116ZRun Repair workload
3502023-09-22T23:23:36.119Z01/30 Write block 490 len 28672 data: 2 2 2 2 2 2 2
3512023-09-22T23:23:36.122Z02/30 Read block 308 len 28672
3522023-09-22T23:23:36.124Z03/30 Write block 170 len 28672 data: 2 2 2 2 2 2 2
3532023-09-22T23:23:36.127Z04/30 Write block 279 len 12288 data: 2 2 2
3542023-09-22T23:23:36.130Z05/30 Write block 22 len 4096 data: 2
3552023-09-22T23:23:36.133Z06/30 Flush
3562023-09-22T23:23:36.136Z07/30 Write block 267 len 40960 data: 2 2 2 2 2 2 2 2 2 2
3572023-09-22T23:23:36.139Z08/30 Write block 122 len 24576 data: 2 2 2 2 2 2
3582023-09-22T23:23:36.142Z09/30 Read block 366 len 16384
3592023-09-22T23:23:36.144Z10/30 Read block 564 len 28672
3602023-09-22T23:23:36.147Z11/30 Write block 252 len 4096 data: 2
3612023-09-22T23:23:36.150Z12/30 Write block 79 len 36864 data: 2 2 2 2 2 2 2 2 2
3622023-09-22T23:23:36.153Z13/30 Read block 503 len 40960
3632023-09-22T23:23:36.155Z14/30 Read block 91 len 32768
3642023-09-22T23:23:36.159Z15/30 Write block 387 len 28672 data: 2 2 2 2 2 2 2
3652023-09-22T23:23:36.162Z16/30 Write block 149 len 12288 data: 2 2 2
3662023-09-22T23:23:36.165Z17/30 Read block 222 len 24576
3672023-09-22T23:23:36.168Z18/30 Write block 55 len 20480 data: 2 2 2 2 2
3682023-09-22T23:23:36.170Z19/30 Read block 133 len 28672
3692023-09-22T23:23:36.173Z20/30 Read block 68 len 20480
3702023-09-22T23:23:36.176Z21/30 Write block 461 len 4096 data: 2
3712023-09-22T23:23:36.179Z22/30 Read block 115 len 12288
3722023-09-22T23:23:36.181Z23/30 Read block 259 len 28672
3732023-09-22T23:23:36.185Z24/30 Read block 249 len 20480
3742023-09-22T23:23:36.188Z25/30 Read block 436 len 20480
3752023-09-22T23:23:36.190Z26/30 Read block 417 len 16384
3762023-09-22T23:23:36.193Z27/30 Write block 127 len 8192 data: 3 2
3772023-09-22T23:23:36.196Z28/30 Read block 368 len 36864
3782023-09-22T23:23:36.199Z29/30 Write block 355 len 24576 data: 2 2 2 2 2 2
3792023-09-22T23:23:36.202Z30/30 Write block 486 len 16384 data: 2 2 2 2
3802023-09-22T23:23:36.205Z----------------------------------------------------------------
3812023-09-22T23:23:36.208Z Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30
3822023-09-22T23:23:36.210ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
3832023-09-22T23:23:36.213Z 1 Acked 1000 Write 7 Done Done Done false
3842023-09-22T23:23:36.216Z 2 Acked 1001 Read 7 Done Done Done false
3852023-09-22T23:23:36.218Z 3 Acked 1002 Write 7 Done Done Done false
3862023-09-22T23:23:36.221Z 4 Acked 1003 Write 3 Done Done Done false
3872023-09-22T23:23:36.224Z 5 Acked 1004 Write 1 Done Done Done false
3882023-09-22T23:23:36.227Z 6 Acked 1005 Flush 0 Done Done Sent false
3892023-09-22T23:23:36.229Z 7 Acked 1006 Write 10 Done Done Sent false
3902023-09-22T23:23:36.232Z 8 Acked 1007 Write 6 Done Done Sent false
3912023-09-22T23:23:36.235Z 9 Acked 1008 Read 4 Done Done Sent false
3922023-09-22T23:23:36.237Z 10 Acked 1009 Read 7 Done Done Sent false
3932023-09-22T23:23:36.240Z 11 Acked 1010 Write 1 Done Done Sent false
3942023-09-22T23:23:36.243Z 12 Acked 1011 Write 9 Done Done Sent false
3952023-09-22T23:23:36.246Z 13 Acked 1012 Read 10 Done Done Sent false
3962023-09-22T23:23:36.249Z 14 Acked 1013 Read 8 Done Done Sent false
3972023-09-22T23:23:36.252Z 15 Acked 1014 Write 7 Done Done Sent false
3982023-09-22T23:23:36.256Z 16 Acked 1015 Write 3 Done Done Sent false
3992023-09-22T23:23:36.259Z 17 Acked 1016 Read 6 Done Done Sent false
4002023-09-22T23:23:36.262Z 18 Acked 1017 Write 5 Done Done Sent false
4012023-09-22T23:23:36.265Z 19 Acked 1018 Read 7 Done Done Sent false
4022023-09-22T23:23:36.268Z 20 Acked 1019 Read 5 Done Done Sent false
4032023-09-22T23:23:36.272Z 21 Acked 1020 Write 1 Done Done Sent false
4042023-09-22T23:23:36.275Z 22 Acked 1021 Read 3 Done Done Sent false
4052023-09-22T23:23:36.278Z 23 Acked 1022 Read 7 Done Done Sent false
4062023-09-22T23:23:36.281Z 24 Acked 1023 Read 5 Done Done Sent false
4072023-09-22T23:23:36.284Z 25 Acked 1024 Read 5 Done Done Sent false
4082023-09-22T23:23:36.287Z 26 Acked 1025 Read 4 Done Done Sent false
4092023-09-22T23:23:36.290Z 27 Acked 1026 Write 2 Done Done Sent false
4102023-09-22T23:23:36.293Z 28 Acked 1027 Read 9 Done Done Sent false
4112023-09-22T23:23:36.297Z 29 Acked 1028 Write 6 Sent Sent Sent false
4122023-09-22T23:23:36.301Z 30 Acked 1029 Write 4 Sent Sent Sent false
4132023-09-22T23:23:36.305Z STATES DS:0 DS:1 DS:2 TOTAL
4142023-09-22T23:23:36.309Z New 0 0 0 0
4152023-09-22T23:23:36.313Z Sent 2 2 25 29
4162023-09-22T23:23:36.316Z Done 28 28 5 61
4172023-09-22T23:23:36.319Z Skipped 0 0 0 0
4182023-09-22T23:23:36.322Z Error 0 0 0 0
4192023-09-22T23:23:36.332ZLast Flush: 1005 1005 0
4202023-09-22T23:23:36.349ZDownstairs last five completed:
4212023-09-22T23:23:36.368ZUpstairs last five completed: 30 29 28 27 26 25
4222023-09-22T23:23:36.378ZWrote out file "/tmp/test_repair_verify.data"
4232023-09-22T23:23:36.385Z
4242023-09-22T23:23:36.391ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
4252023-09-22T23:23:36.398Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.948143852Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1165}
4262023-09-22T23:23:36.401Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.948435646Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1165}
4272023-09-22T23:23:36.405Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.948460111Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1165}
4282023-09-22T23:23:36.407Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.983578091Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1165}
4292023-09-22T23:23:36.410Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.983612435Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1165}
4302023-09-22T23:23:36.414Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:34.983631579Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1165}
4312023-09-22T23:23:36.418Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:35.0104131Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1165}
4322023-09-22T23:23:36.421Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:35.010451198Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1165}
4332023-09-22T23:23:36.424Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:35.010471123Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1165}
4342023-09-22T23:23:36.427ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4352023-09-22T23:23:36.430Z 0 000-019 1 1 1 1 1 1 F F F
4362023-09-22T23:23:36.433Z 1 020-039 2 2 1 2 2 1 F F T <---
4372023-09-22T23:23:36.436Z 2 040-059 1 1 1 1 1 1 T T F <---
4382023-09-22T23:23:36.442Z 3 060-079 1 1 1 1 1 1 T T F <---
4392023-09-22T23:23:36.446Z 4 080-099 1 1 1 1 1 1 T T F <---
4402023-09-22T23:23:36.450Z 5 100-119 1 1 1 1 1 1 F F F
4412023-09-22T23:23:36.453Z 6 120-139 1 1 1 1 1 1 T T F <---
4422023-09-22T23:23:36.456Z 7 140-159 1 1 1 1 1 1 T T F <---
4432023-09-22T23:23:36.459Z 8 160-179 2 2 1 2 2 1 F F T <---
4442023-09-22T23:23:36.462Z 9 180-199 1 1 1 1 1 1 F F F
4452023-09-22T23:23:36.466Z 10 200-219 1 1 1 1 1 1 F F F
4462023-09-22T23:23:36.470Z 11 220-239 1 1 1 1 1 1 F F F
4472023-09-22T23:23:36.473Z 12 240-259 1 1 1 1 1 1 T T F <---
4482023-09-22T23:23:36.476Z 13 260-279 2 2 1 2 2 1 T T T <---
4492023-09-22T23:23:36.479Z 14 280-299 2 2 1 2 2 1 F F T <---
4502023-09-22T23:23:36.483Z 15 300-319 1 1 1 1 1 1 F F F
4512023-09-22T23:23:36.486Z 16 320-339 1 1 1 1 1 1 F F F
4522023-09-22T23:23:36.489Z 17 340-359 1 1 1 1 1 1 T T F <---
4532023-09-22T23:23:36.492Z 18 360-379 1 1 1 1 1 1 T T F <---
4542023-09-22T23:23:36.495Z 19 380-399 1 1 1 1 1 1 T T F <---
4552023-09-22T23:23:36.497Z 20 400-419 1 1 1 1 1 1 F F F
4562023-09-22T23:23:36.508Z 21 420-439 1 1 1 1 1 1 F F F
4572023-09-22T23:23:36.512Z 22 440-459 1 1 1 1 1 1 F F F
4582023-09-22T23:23:36.516Z 23 460-479 1 1 1 1 1 1 T T F <---
4592023-09-22T23:23:36.521Z 24 480-499 2 2 1 2 2 1 T T T <---
4602023-09-22T23:23:36.526ZError: Difference in extent metadata found!
4612023-09-22T23:23:36.529Z 25 500-519 1 1 1 1 1 1 F F F
4622023-09-22T23:23:36.532Z 26 520-539 1 1 1 1 1 1 F F F
4632023-09-22T23:23:36.535Z 27 540-559 1 1 1 1 1 1 F F F
4642023-09-22T23:23:36.537Z 28 560-579 1 1 1 1 1 1 F F F
4652023-09-22T23:23:36.540Z 29 580-599 1 1 1 1 1 1 F F F
4662023-09-22T23:23:36.543ZMax gen: 2, Max flush: 2
4672023-09-22T23:23:36.545ZA Difference in extent metadata is expected here
4682023-09-22T23:23:36.548ZOn loop 1
4692023-09-22T23:23:36.551Z
4702023-09-22T23:23:36.553ZVerifying data now
4712023-09-22T23:23:42.879ZNo change after verify
4722023-09-22T23:23:42.890ZLoop: 1 Downstairs dump after verify (and repair):
4732023-09-22T23:23:42.898Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.794832073Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1170}
4742023-09-22T23:23:42.901Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.795245949Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1170}
4752023-09-22T23:23:42.904Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.795270831Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1170}
4762023-09-22T23:23:42.917Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.813960728Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1170}
4772023-09-22T23:23:42.920Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.814022412Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1170}
4782023-09-22T23:23:42.923Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.814045191Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1170}
4792023-09-22T23:23:42.928Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.824931122Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1170}
4802023-09-22T23:23:42.930Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.824982474Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1170}
4812023-09-22T23:23:42.934Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.82501008Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1170}
4822023-09-22T23:23:42.939ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
4832023-09-22T23:23:42.944Z 0 000-019 1 1 1 1 1 1 F F F
4842023-09-22T23:23:42.950Z 1 020-039 2 2 2 2 2 2 F F F
4852023-09-22T23:23:42.957Z 2 040-059 3 3 3 3 3 3 F F F
4862023-09-22T23:23:42.965Z 3 060-079 3 3 3 3 3 3 F F F
4872023-09-22T23:23:42.976Z 4 080-099 3 3 3 3 3 3 F F F
4882023-09-22T23:23:42.996Z 5 100-119 1 1 1 1 1 1 F F F
4892023-09-22T23:23:43.020Z 6 120-139 3 3 3 3 3 3 F F F
4902023-09-22T23:23:43.033Z 7 140-159 3 3 3 3 3 3 F F F
4912023-09-22T23:23:43.046Z 8 160-179 2 2 2 2 2 2 F F F
4922023-09-22T23:23:43.056Z 9 180-199 1 1 1 1 1 1 F F F
4932023-09-22T23:23:43.065Z 10 200-219 1 1 1 1 1 1 F F F
4942023-09-22T23:23:43.075Z 11 220-239 1 1 1 1 1 1 F F F
4952023-09-22T23:23:43.082Z 12 240-259 3 3 3 3 3 3 F F F
4962023-09-22T23:23:43.087Z 13 260-279 3 3 3 3 3 3 F F F
4972023-09-22T23:23:43.095Z 14 280-299 2 2 2 2 2 2 F F F
4982023-09-22T23:23:43.098Z 15 300-319 1 1 1 1 1 1 F F F
4992023-09-22T23:23:43.101Z 16 320-339 1 1 1 1 1 1 F F F
5002023-09-22T23:23:43.104Z 17 340-359 3 3 3 3 3 3 F F F
5012023-09-22T23:23:43.107Z 18 360-379 3 3 3 3 3 3 F F F
5022023-09-22T23:23:43.110Z 19 380-399 3 3 3 3 3 3 F F F
5032023-09-22T23:23:43.113Z 20 400-419 1 1 1 1 1 1 F F F
5042023-09-22T23:23:43.116Z 21 420-439 1 1 1 1 1 1 F F F
5052023-09-22T23:23:43.121Z 22 440-459 1 1 1 1 1 1 F F F
5062023-09-22T23:23:43.123Z 23 460-479 3 3 3 3 3 3 F F F
5072023-09-22T23:23:43.126Z 24 480-499 3 3 3 3 3 3 F F F
5082023-09-22T23:23:43.129Z 25 500-519 1 1 1 1 1 1 F F F
5092023-09-22T23:23:43.132Z 26 520-539 1 1 1 1 1 1 F F F
5102023-09-22T23:23:43.135Z 27 540-559 1 1 1 1 1 1 F F F
5112023-09-22T23:23:43.138Z 28 560-579 1 1 1 1 1 1 F F F
5122023-09-22T23:23:43.141Z 29 580-599 1 1 1 1 1 1 F F F
5132023-09-22T23:23:43.144ZMax gen: 3, Max flush: 3
5142023-09-22T23:23:43.146Z
5152023-09-22T23:23:43.149ZBegin loop 2
5162023-09-22T23:23:43.152ZDownstairs to restart: 2
5172023-09-22T23:23:43.155Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.87315683Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5182023-09-22T23:23:43.158Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.873534633Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5192023-09-22T23:23:43.161Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.873588881Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5202023-09-22T23:23:43.164Z{"msg":"Crucible stats registered with UUID: 9ae99ece-ef47-4eb0-867e-9ebe07a65c1a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.87361479Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5212023-09-22T23:23:43.167Z{"msg":"Crucible 9ae99ece-ef47-4eb0-867e-9ebe07a65c1a has session id: 6a13728b-9960-4808-a641-dbeb0fd1aedd","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.873655571Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5222023-09-22T23:23:43.170ZCrucible runtime is spawned
5232023-09-22T23:23:43.173ZThe guest has requested activation with gen:4
5242023-09-22T23:23:43.176Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.873953473Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172,"task":"up_listen"}
5252023-09-22T23:23:43.179Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.87402999Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5262023-09-22T23:23:43.182Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.874074637Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5272023-09-22T23:23:43.184Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.874113727Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172,"looper":"2"}
5282023-09-22T23:23:43.187Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.874332897Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172,"looper":"1"}
5292023-09-22T23:23:43.190Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.874558519Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172,"looper":"0"}
5302023-09-22T23:23:43.193Z{"msg":"[1] 9ae99ece-ef47-4eb0-867e-9ebe07a65c1a looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.874627542Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172,"looper":"1"}
5312023-09-22T23:23:43.196Z{"msg":"[0] 9ae99ece-ef47-4eb0-867e-9ebe07a65c1a looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.874805786Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172,"looper":"0"}
5322023-09-22T23:23:43.199Z{"msg":"9ae99ece-ef47-4eb0-867e-9ebe07a65c1a active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.875051949Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5332023-09-22T23:23:43.203Z{"msg":"Set desired generation to :4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.875109598Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5342023-09-22T23:23:43.206Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.875154793Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5352023-09-22T23:23:43.209Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.875475373Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5362023-09-22T23:23:43.212Z{"msg":"[1] 9ae99ece-ef47-4eb0-867e-9ebe07a65c1a (047d0409-6c5a-4a5a-a6b0-787428127bb9) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.875935549Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5372023-09-22T23:23:43.215Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.876005717Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5382023-09-22T23:23:43.218Z{"msg":"[1] client is_active_req TRUE, promote! session 047d0409-6c5a-4a5a-a6b0-787428127bb9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.87605019Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5392023-09-22T23:23:43.221Z{"msg":"[0] 9ae99ece-ef47-4eb0-867e-9ebe07a65c1a (047d0409-6c5a-4a5a-a6b0-787428127bb9) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.87620764Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5402023-09-22T23:23:43.224Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.876271434Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5412023-09-22T23:23:43.227Z{"msg":"[0] client is_active_req TRUE, promote! session 047d0409-6c5a-4a5a-a6b0-787428127bb9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.876321148Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5422023-09-22T23:23:43.230Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.876547899Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5432023-09-22T23:23:43.233Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.876609552Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5442023-09-22T23:23:43.236Z{"msg":"9ae99ece-ef47-4eb0-867e-9ebe07a65c1a WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.876652265Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5452023-09-22T23:23:43.240Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.876777727Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5462023-09-22T23:23:43.243Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.87684988Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5472023-09-22T23:23:43.246Z{"msg":"9ae99ece-ef47-4eb0-867e-9ebe07a65c1a WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.87692429Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5482023-09-22T23:23:43.249Z{"msg":"[1] 9ae99ece-ef47-4eb0-867e-9ebe07a65c1a (047d0409-6c5a-4a5a-a6b0-787428127bb9) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.878834511Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5492023-09-22T23:23:43.252Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.878901624Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5502023-09-22T23:23:43.255Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:23:41.879020518Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5512023-09-22T23:23:43.258Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.879074179Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5522023-09-22T23:23:43.261Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.879130122Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5532023-09-22T23:23:43.263Z{"msg":"9ae99ece-ef47-4eb0-867e-9ebe07a65c1a WaitActive WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.879184843Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5542023-09-22T23:23:43.266Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.87921378Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5552023-09-22T23:23:43.269Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.879232624Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5562023-09-22T23:23:43.272Z{"msg":"[0] 9ae99ece-ef47-4eb0-867e-9ebe07a65c1a (047d0409-6c5a-4a5a-a6b0-787428127bb9) WaitActive WaitQuorum New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.879253402Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5572023-09-22T23:23:43.275Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.879271572Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5582023-09-22T23:23:43.277Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:23:41.879289178Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5592023-09-22T23:23:43.280Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.879306832Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5602023-09-22T23:23:43.283Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.879324835Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5612023-09-22T23:23:43.286Z{"msg":"9ae99ece-ef47-4eb0-867e-9ebe07a65c1a WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.879343232Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5622023-09-22T23:23:43.289Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.879361079Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5632023-09-22T23:23:43.291Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:41.879378154Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5642023-09-22T23:23:43.979Z{"msg":"[2] 9ae99ece-ef47-4eb0-867e-9ebe07a65c1a looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.875362067Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172,"looper":"2"}
5652023-09-22T23:23:43.982Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.875444156Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5662023-09-22T23:23:43.985Z{"msg":"[2] 9ae99ece-ef47-4eb0-867e-9ebe07a65c1a (047d0409-6c5a-4a5a-a6b0-787428127bb9) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.876605491Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5672023-09-22T23:23:43.988Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.87665946Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5682023-09-22T23:23:43.991Z{"msg":"[2] client is_active_req TRUE, promote! session 047d0409-6c5a-4a5a-a6b0-787428127bb9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.876683612Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5692023-09-22T23:23:43.994Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.877264913Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5702023-09-22T23:23:43.997Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.87732272Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5712023-09-22T23:23:44.001Z{"msg":"9ae99ece-ef47-4eb0-867e-9ebe07a65c1a WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.877365658Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5722023-09-22T23:23:44.011Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.879703141Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5732023-09-22T23:23:44.035Z{"msg":"[2] 9ae99ece-ef47-4eb0-867e-9ebe07a65c1a (047d0409-6c5a-4a5a-a6b0-787428127bb9) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.879812128Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5742023-09-22T23:23:44.459Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.879883508Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5752023-09-22T23:23:44.462Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:23:42.879953698Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5762023-09-22T23:23:44.465Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.880038655Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5772023-09-22T23:23:44.469Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.88011735Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5782023-09-22T23:23:44.472Z{"msg":"9ae99ece-ef47-4eb0-867e-9ebe07a65c1a WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.880183533Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5792023-09-22T23:23:44.475Z{"msg":"[0]R flush_numbers[0..12]: [1, 2, 3, 3, 3, 1, 3, 3, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.88024851Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5802023-09-22T23:23:44.478Z{"msg":"[0]R generation[0..12]: [1, 2, 3, 3, 3, 1, 3, 3, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.880315325Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5812023-09-22T23:23:44.482Z{"msg":"The guest has finished waiting for activation with:4
5822023-09-22T23:23:44.498ZWait for a query_work_queue command to finish before sending IO
5832023-09-22T23:23:44.507Z[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.880573123Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5842023-09-22T23:23:44.513Z{"msg":"[1]R flush_numbers[0..12]: [1, 2, 3, 3, 3, 1, 3, 3, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.880623259Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5852023-09-22T23:23:44.522Z{"msg":"[1]R generation[0..12]: [1, 2, 3, 3, 3, 1, 3, 3, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.880725028Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5862023-09-22T23:23:44.529Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.880793147Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5872023-09-22T23:23:44.532Z{"msg":"[2]R flush_numbers[0..12]: [1, 2, 3, 3, 3, 1, 3, 3, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.88086765Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5882023-09-22T23:23:44.540Z{"msg":"[2]R generation[0..12]: [1, 2, 3, 3, 3, 1, 3, 3, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.880930482Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5892023-09-22T23:23:44.544Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.881032131Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5902023-09-22T23:23:44.551Z{"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.881099199Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5912023-09-22T23:23:44.557Z{"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.881142588Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5922023-09-22T23:23:44.563Z{"msg":"Next flush: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.881202371Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5932023-09-22T23:23:44.567Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.881266087Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5942023-09-22T23:23:44.571Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.881314481Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5952023-09-22T23:23:44.574Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.881369306Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5962023-09-22T23:23:44.590Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.881437335Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5972023-09-22T23:23:44.598Z{"msg":"9ae99ece-ef47-4eb0-867e-9ebe07a65c1a is now active with session: 047d0409-6c5a-4a5a-a6b0-787428127bb9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.881488588Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
5982023-09-22T23:23:44.603Z{"msg":"9ae99ece-ef47-4eb0-867e-9ebe07a65c1a Set Active after no repair","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
5992023-09-22T23:23:44.610Z,"time":"2023-09-22T23:23:42.881544608Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
6002023-09-22T23:23:44.616Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.88166908Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
6012023-09-22T23:23:44.621Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.881735605Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
6022023-09-22T23:23:44.637Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.881801128Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
6032023-09-22T23:23:44.642Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.881864093Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
6042023-09-22T23:23:44.649Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.881925038Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
6052023-09-22T23:23:44.654Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.882013765Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
6062023-09-22T23:23:44.659Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.882153097Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
6072023-09-22T23:23:44.664Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.882197225Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
6082023-09-22T23:23:44.667Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.88225676Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
6092023-09-22T23:23:44.670Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.882321901Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1172}
6102023-09-22T23:23:44.673Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.882356166Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":Loading write count information from file "/tmp/test_repair_verify.data"
6112023-09-22T23:23:44.676Z1172}
6122023-09-22T23:23:44.679ZRun Repair workload
6132023-09-22T23:23:44.682Z01/30 Write block 488 len 8192 data: 3 3
6142023-09-22T23:23:44.685Z02/30 Read block 81 len 4096
6152023-09-22T23:23:44.688Z03/30 Read block 336 len 8192
6162023-09-22T23:23:44.692Z04/30 Flush
6172023-09-22T23:23:44.694Z05/30 Flush
6182023-09-22T23:23:44.697Z06/30 Write block 158 len 20480 data: 2 2 2 2 2
6192023-09-22T23:23:44.700Z07/30 Write block 67 len 24576 data: 2 2 2 2 2 2
6202023-09-22T23:23:44.704Z08/30 Write block 43 len 36864 data: 2 2 2 2 2 2 2 2 2
6212023-09-22T23:23:44.707Z09/30 Write block 450 len 4096 data: 2
6222023-09-22T23:23:44.710Z10/30 Read block 583 len 24576
6232023-09-22T23:23:44.714Z11/30 Read block 52 len 40960
6242023-09-22T23:23:44.717Z12/30 Flush
6252023-09-22T23:23:44.720Z13/30 Write block 321 len 32768 data: 2 2 2 2 2 2 2 2
6262023-09-22T23:23:44.724Z14/30 Write block 580 len 24576 data: 2 2 2 2 2 2
6272023-09-22T23:23:44.727Z15/30 Read block 88 len 20480
6282023-09-22T23:23:44.730Z16/30 Read block 292 len 20480
6292023-09-22T23:23:44.733Z17/30 Read block 129 len 32768
6302023-09-22T23:23:44.736Z18/30 Read block 170 len 40960
6312023-09-22T23:23:44.738Z19/30 Read block 501 len 8192
6322023-09-22T23:23:44.741Z20/30 Write block 409 len 24576 data: 2 2 2 2 2 2
6332023-09-22T23:23:44.745Z21/30 Write block 132 len 4096 data: 2
6342023-09-22T23:23:44.748Z22/30 Read block 412 len 32768
6352023-09-22T23:23:44.751Z23/30 Read block 41 len 32768
6362023-09-22T23:23:44.754Z24/30 Write block 489 len 20480 data: 4 3 3 3 3
6372023-09-22T23:23:44.757Z25/30 Read block 159 len 16384
6382023-09-22T23:23:44.759Z26/30 Write block 549 len 20480 data: 2 2 2 2 2
6392023-09-22T23:23:44.763Z27/30 Read block 597 len 4096
6402023-09-22T23:23:44.766ZError: Difference in extent metadata found!
6412023-09-22T23:23:44.769Z28/30 Write block 14 len 8192 data: 2 2
6422023-09-22T23:23:44.772Z29/30 Write block 142 len 16384 data: 2 2 2 2
6432023-09-22T23:23:44.775Z30/30 Write block 465 len 20480 data: 2 2 2 2 2
6442023-09-22T23:23:44.778Z----------------------------------------------------------------
6452023-09-22T23:23:44.781Z Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:25
6462023-09-22T23:23:44.784ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
6472023-09-22T23:23:44.788Z 6 Acked 1005 Write 5 Done Done Sent false
6482023-09-22T23:23:44.791Z 7 Acked 1006 Write 6 Done Done Sent false
6492023-09-22T23:23:44.795Z 8 Acked 1007 Write 9 Done Done Sent false
6502023-09-22T23:23:44.798Z 9 Acked 1008 Write 1 Done Done Sent false
6512023-09-22T23:23:44.801Z 10 Acked 1009 Read 6 Done Done Sent false
6522023-09-22T23:23:44.804Z 11 Acked 1010 Read 10 Done Done Sent false
6532023-09-22T23:23:44.807Z 12 Acked 1011 Flush 0 Done Done Sent false
6542023-09-22T23:23:44.810Z 13 Acked 1012 Write 8 Done Done Sent false
6552023-09-22T23:23:44.812Z 14 Acked 1013 Write 6 Done Done Sent false
6562023-09-22T23:23:44.815Z 15 Acked 1014 Read 5 Done Done Sent false
6572023-09-22T23:23:44.818Z 16 Acked 1015 Read 5 Done Done Sent false
6582023-09-22T23:23:44.821Z 17 Acked 1016 Read 8 Done Done Sent false
6592023-09-22T23:23:44.825Z 18 Acked 1017 Read 10 Done Done Sent false
6602023-09-22T23:23:44.828Z 19 Acked 1018 Read 2 Done Done Sent false
6612023-09-22T23:23:44.831Z 20 Acked 1019 Write 6 Done Done Sent false
6622023-09-22T23:23:44.834Z 21 Acked 1020 Write 1 Done Done Sent false
6632023-09-22T23:23:44.837Z 22 Acked 1021 Read 8 Done Done Sent false
6642023-09-22T23:23:44.839Z 23 Acked 1022 Read 8 Done Done Sent false
6652023-09-22T23:23:44.842Z 24 Acked 1023 Write 5 Done Done Sent false
6662023-09-22T23:23:44.845Z 25 Acked 1024 Read 4 Done Done Sent false
6672023-09-22T23:23:44.848Z 26 Acked 1025 Write 5 Done Done Sent false
6682023-09-22T23:23:44.850Z 27 Acked 1026 Read 1 Done Done Sent false
6692023-09-22T23:23:44.854Z 28 Acked 1027 Write 2 Done Done Sent false
6702023-09-22T23:23:44.858Z 29 Acked 1028 Write 4 Sent Sent Sent false
6712023-09-22T23:23:44.861Z 30 Acked 1029 Write 5 Sent Sent Sent false
6722023-09-22T23:23:44.864Z STATES DS:0 DS:1 DS:2 TOTAL
6732023-09-22T23:23:44.866Z New 0 0 0 0
6742023-09-22T23:23:44.869Z Sent 2 2 25 29
6752023-09-22T23:23:44.873Z Done 23 23 0 46
6762023-09-22T23:23:44.876Z Skipped 0 0 0 0
6772023-09-22T23:23:44.878Z Error 0 0 0 0
6782023-09-22T23:23:44.881ZLast Flush: 1011 1011 1004
6792023-09-22T23:23:44.884ZDownstairs last five completed: 1004 1003 1002 1001 1000
6802023-09-22T23:23:44.887ZUpstairs last five completed: 30 29 28 27 26 25
6812023-09-22T23:23:44.891ZWrote out file "/tmp/test_repair_verify.data"
6822023-09-22T23:23:44.894Z
6832023-09-22T23:23:44.897ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
6842023-09-22T23:23:44.900Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.999332854Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1173}
6852023-09-22T23:23:44.903Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.999579749Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1173}
6862023-09-22T23:23:44.906Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:42.999602773Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1173}
6872023-09-22T23:23:44.909Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:43.031450817Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1173}
6882023-09-22T23:23:44.912Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:43.031482412Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1173}
6892023-09-22T23:23:44.915Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:43.031501139Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1173}
6902023-09-22T23:23:44.919Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:43.056750929Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1173}
6912023-09-22T23:23:44.922Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:43.056780215Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1173}
6922023-09-22T23:23:44.924Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:43.056798643Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1173}
6932023-09-22T23:23:44.927ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
6942023-09-22T23:23:44.931Z 0 000-019 1 1 1 1 1 1 T T F <---
6952023-09-22T23:23:44.934Z 1 020-039 2 2 2 2 2 2 F F F
6962023-09-22T23:23:44.937Z 2 040-059 4 4 3 6 6 3 F F F <---
6972023-09-22T23:23:44.952Z 3 060-079 4 4 3 6 6 3 F F F <---
6982023-09-22T23:23:44.955Z 4 080-099 3 3 3 3 3 3 F F F
6992023-09-22T23:23:44.958Z 5 100-119 1 1 1 1 1 1 F F F
7002023-09-22T23:23:44.960Z 6 120-139 3 3 3 3 3 3 T T F <---
7012023-09-22T23:23:44.963Z 7 140-159 4 4 3 6 6 3 T T F <---
7022023-09-22T23:23:44.966Z 8 160-179 4 4 2 6 6 2 F F F <---
7032023-09-22T23:23:44.969Z 9 180-199 1 1 1 1 1 1 F F F
7042023-09-22T23:23:44.972Z 10 200-219 1 1 1 1 1 1 F F F
7052023-09-22T23:23:44.975Z 11 220-239 1 1 1 1 1 1 F F F
7062023-09-22T23:23:44.978Z 12 240-259 3 3 3 3 3 3 F F F
7072023-09-22T23:23:44.981Z 13 260-279 3 3 3 3 3 3 F F F
7082023-09-22T23:23:44.984Z 14 280-299 2 2 2 2 2 2 F F F
7092023-09-22T23:23:44.987Z 15 300-319 1 1 1 1 1 1 F F F
7102023-09-22T23:23:44.990Z 16 320-339 1 1 1 1 1 1 T T F <---
7112023-09-22T23:23:44.993Z 17 340-359 3 3 3 3 3 3 F F F
7122023-09-22T23:23:44.996Z 18 360-379 3 3 3 3 3 3 F F F
7132023-09-22T23:23:44.998Z 19 380-399 3 3 3 3 3 3 F F F
7142023-09-22T23:23:45.001Z 20 400-419 1 1 1 1 1 1 T T F <---
7152023-09-22T23:23:45.004Z 21 420-439 1 1 1 1 1 1 F F F
7162023-09-22T23:23:45.007Z 22 440-459 4 4 1 6 6 1 F F F <---
7172023-09-22T23:23:45.010Z 23 460-479 3 3 3 3 3 3 T T F <---
7182023-09-22T23:23:45.013Z 24 480-499 4 4 4 4 4 4 T T F <---
7192023-09-22T23:23:45.016Z 25 500-519 1 1 1 1 1 1 F F F
7202023-09-22T23:23:45.018Z 26 520-539 1 1 1 1 1 1 F F F
7212023-09-22T23:23:45.021Z 27 540-559 1 1 1 1 1 1 T T F <---
7222023-09-22T23:23:45.024Z 28 560-579 1 1 1 1 1 1 F F F
7232023-09-22T23:23:45.026Z 29 580-599 1 1 1 1 1 1 T T F <---
7242023-09-22T23:23:45.030ZMax gen: 4, Max flush: 6
7252023-09-22T23:23:45.033ZA Difference in extent metadata is expected here
7262023-09-22T23:23:45.036ZOn loop 2
7272023-09-22T23:23:45.038Z
7282023-09-22T23:23:45.041ZVerifying data now
7292023-09-22T23:23:50.609ZNo change after verify
7302023-09-22T23:23:50.612ZLoop: 2 Downstairs dump after verify (and repair):
7312023-09-22T23:23:50.620Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.517125898Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1178}
7322023-09-22T23:23:50.623Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.517511923Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1178}
7332023-09-22T23:23:50.627Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.517536096Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1178}
7342023-09-22T23:23:50.639Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.536214986Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1178}
7352023-09-22T23:23:50.642Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.53626856Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1178}
7362023-09-22T23:23:50.644Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.536289055Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1178}
7372023-09-22T23:23:50.650Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.547264655Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1178}
7382023-09-22T23:23:50.653Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.547316987Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1178}
7392023-09-22T23:23:50.655Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.547337327Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1178}
7402023-09-22T23:23:50.661ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
7412023-09-22T23:23:50.664Z 0 000-019 5 5 5 7 7 7 F F F
7422023-09-22T23:23:50.667Z 1 020-039 2 2 2 2 2 2 F F F
7432023-09-22T23:23:50.670Z 2 040-059 4 4 4 6 6 6 F F F
7442023-09-22T23:23:50.673Z 3 060-079 4 4 4 6 6 6 F F F
7452023-09-22T23:23:50.676Z 4 080-099 3 3 3 3 3 3 F F F
7462023-09-22T23:23:50.679Z 5 100-119 1 1 1 1 1 1 F F F
7472023-09-22T23:23:50.682Z 6 120-139 5 5 5 7 7 7 F F F
7482023-09-22T23:23:50.686Z 7 140-159 5 5 5 7 7 7 F F F
7492023-09-22T23:23:50.689Z 8 160-179 4 4 4 6 6 6 F F F
7502023-09-22T23:23:50.692Z 9 180-199 1 1 1 1 1 1 F F F
7512023-09-22T23:23:50.695Z 10 200-219 1 1 1 1 1 1 F F F
7522023-09-22T23:23:50.698Z 11 220-239 1 1 1 1 1 1 F F F
7532023-09-22T23:23:50.701Z 12 240-259 3 3 3 3 3 3 F F F
7542023-09-22T23:23:50.704Z 13 260-279 3 3 3 3 3 3 F F F
7552023-09-22T23:23:50.707Z 14 280-299 2 2 2 2 2 2 F F F
7562023-09-22T23:23:50.710Z 15 300-319 1 1 1 1 1 1 F F F
7572023-09-22T23:23:50.713Z 16 320-339 5 5 5 7 7 7 F F F
7582023-09-22T23:23:50.716Z 17 340-359 3 3 3 3 3 3 F F F
7592023-09-22T23:23:50.719Z 18 360-379 3 3 3 3 3 3 F F F
7602023-09-22T23:23:50.722Z 19 380-399 3 3 3 3 3 3 F F F
7612023-09-22T23:23:50.725Z 20 400-419 5 5 5 7 7 7 F F F
7622023-09-22T23:23:50.728Z 21 420-439 1 1 1 1 1 1 F F F
7632023-09-22T23:23:50.730Z 22 440-459 4 4 4 6 6 6 F F F
7642023-09-22T23:23:50.733Z 23 460-479 5 5 5 7 7 7 F F F
7652023-09-22T23:23:50.736Z 24 480-499 5 5 5 7 7 7 F F F
7662023-09-22T23:23:50.739Z 25 500-519 1 1 1 1 1 1 F F F
7672023-09-22T23:23:50.742Z 26 520-539 1 1 1 1 1 1 F F F
7682023-09-22T23:23:50.745Z 27 540-559 5 5 5 7 7 7 F F F
7692023-09-22T23:23:50.747Z 28 560-579 1 1 1 1 1 1 F F F
7702023-09-22T23:23:50.750Z 29 580-599 5 5 5 7 7 7 F F F
7712023-09-22T23:23:50.754ZMax gen: 5, Max flush: 7
7722023-09-22T23:23:50.756Z
7732023-09-22T23:23:50.759ZBegin loop 3
7742023-09-22T23:23:50.762ZDownstairs to restart: 2
7752023-09-22T23:23:50.765Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.595133138Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
7762023-09-22T23:23:50.768Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.595505473Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
7772023-09-22T23:23:50.771Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.59555736Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
7782023-09-22T23:23:50.774Z{"msg":"Crucible stats registered with UUID: 0aee7db3-f87e-48bd-8427-901f3b745d1e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.595601295Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
7792023-09-22T23:23:50.777Z{"msg":"Crucible 0aee7db3-f87e-48bd-8427-901f3b745d1e has session id: ea142559-29a4-4e1b-80d3-9116ccdb3e11","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.595659494Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
7802023-09-22T23:23:50.780ZCrucible runtime is spawned
7812023-09-22T23:23:50.783Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.596007237Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180,"task":"up_listen"}
7822023-09-22T23:23:50.788Z{"msg":"The guest has requested activation with gen:6
7832023-09-22T23:23:50.791ZWait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.596077636Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
7842023-09-22T23:23:50.794Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.596120783Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
7852023-09-22T23:23:50.797Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.596159606Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180,"looper":"0"}
7862023-09-22T23:23:50.801Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.596496823Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180,"looper":"1"}
7872023-09-22T23:23:50.803Z{"msg":"[0] 0aee7db3-f87e-48bd-8427-901f3b745d1e looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.596584841Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180,"looper":"0"}
7882023-09-22T23:23:50.807Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.596663534Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180,"looper":"2"}
7892023-09-22T23:23:50.810Z{"msg":"[1] 0aee7db3-f87e-48bd-8427-901f3b745d1e looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.596730261Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180,"looper":"1"}
7902023-09-22T23:23:50.813Z{"msg":"0aee7db3-f87e-48bd-8427-901f3b745d1e active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.59712999Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
7912023-09-22T23:23:50.816Z{"msg":"Set desired generation to :6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.597191506Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
7922023-09-22T23:23:50.818Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.59721973Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
7932023-09-22T23:23:50.821Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.597581124Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
7942023-09-22T23:23:50.824Z{"msg":"[0] 0aee7db3-f87e-48bd-8427-901f3b745d1e (d1c80d48-e95f-4486-82fc-5697ca19e314) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.597862751Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
7952023-09-22T23:23:50.831Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.597919719Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
7962023-09-22T23:23:50.834Z{"msg":"[0] client is_active_req TRUE, promote! session d1c80d48-e95f-4486-82fc-5697ca19e314","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.597962342Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
7972023-09-22T23:23:50.837Z{"msg":"[1] 0aee7db3-f87e-48bd-8427-901f3b745d1e (d1c80d48-e95f-4486-82fc-5697ca19e314) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.597987168Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
7982023-09-22T23:23:50.840Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.598061543Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
7992023-09-22T23:23:50.843Z{"msg":"[1] client is_active_req TRUE, promote! session d1c80d48-e95f-4486-82fc-5697ca19e314","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.598140916Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8002023-09-22T23:23:50.845Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.598492098Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8012023-09-22T23:23:50.848Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.59855254Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8022023-09-22T23:23:50.851Z{"msg":"0aee7db3-f87e-48bd-8427-901f3b745d1e WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.598594865Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8032023-09-22T23:23:50.854Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.598743065Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8042023-09-22T23:23:50.857Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.598835336Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8052023-09-22T23:23:50.864Z{"msg":"0aee7db3-f87e-48bd-8427-901f3b745d1e WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.598911302Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8062023-09-22T23:23:50.868Z{"msg":"[0] 0aee7db3-f87e-48bd-8427-901f3b745d1e (d1c80d48-e95f-4486-82fc-5697ca19e314) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.60076167Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8072023-09-22T23:23:50.871Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.600820599Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8082023-09-22T23:23:50.874Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:23:49.600843384Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8092023-09-22T23:23:50.877Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.60088117Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8102023-09-22T23:23:50.880Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.601172456Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8112023-09-22T23:23:50.883Z{"msg":"0aee7db3-f87e-48bd-8427-901f3b745d1e WaitQuorum WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.601250909Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8122023-09-22T23:23:50.887Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.601322906Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8132023-09-22T23:23:50.890Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.601391334Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8142023-09-22T23:23:50.893Z{"msg":"[1] 0aee7db3-f87e-48bd-8427-901f3b745d1e (d1c80d48-e95f-4486-82fc-5697ca19e314) WaitQuorum WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.601467661Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8152023-09-22T23:23:50.896Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.601543662Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8162023-09-22T23:23:50.899Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:23:49.601593532Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8172023-09-22T23:23:50.903Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.601632575Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8182023-09-22T23:23:50.906Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.601671427Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8192023-09-22T23:23:50.909Z{"msg":"0aee7db3-f87e-48bd-8427-901f3b745d1e WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.601721571Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8202023-09-22T23:23:50.912Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.60176194Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8212023-09-22T23:23:50.915Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:49.601781302Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8222023-09-22T23:23:51.702Z{"msg":"[2] 0aee7db3-f87e-48bd-8427-901f3b745d1e looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.598655917Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180,"looper":"2"}
8232023-09-22T23:23:51.705Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.59876805Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8242023-09-22T23:23:51.708Z{"msg":"[2] 0aee7db3-f87e-48bd-8427-901f3b745d1e (d1c80d48-e95f-4486-82fc-5697ca19e314) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.599949536Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8252023-09-22T23:23:51.711Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.600008445Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8262023-09-22T23:23:51.714Z{"msg":"[2] client is_active_req TRUE, promote! session d1c80d48-e95f-4486-82fc-5697ca19e314","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.600033105Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8272023-09-22T23:23:51.717Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.600694005Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8282023-09-22T23:23:51.720Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.600754885Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8292023-09-22T23:23:51.723Z{"msg":"0aee7db3-f87e-48bd-8427-901f3b745d1e WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.600796784Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8302023-09-22T23:23:51.726Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.602967203Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8312023-09-22T23:23:51.729Z{"msg":"[2] 0aee7db3-f87e-48bd-8427-901f3b745d1e (d1c80d48-e95f-4486-82fc-5697ca19e314) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.603268441Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8322023-09-22T23:23:51.732Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.603335846Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8332023-09-22T23:23:51.735Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:23:50.603357087Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8342023-09-22T23:23:51.738Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.603374946Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8352023-09-22T23:23:51.741Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.60339292Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8362023-09-22T23:23:51.744Z{"msg":"0aee7db3-f87e-48bd-8427-901f3b745d1e WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.603475079Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8372023-09-22T23:23:51.747Z{"msg":"[0]R flush_numbers[0..12]: [7, 2, 6, 6, 3, 1, 7, 7, 6, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.603515851Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8382023-09-22T23:23:51.750Z{"msg":"[0]R generation[0..12]: [5, 2, 4, 4, 3, 1, 5, 5, 4, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.603536896Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8392023-09-22T23:23:51.753Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.60357509Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8402023-09-22T23:23:51.762Z{"msg":"[1]R flush_numbers[0..12]: [7, 2, 6, 6, 3, 1, 7, 7, 6, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.603623294Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8412023-09-22T23:23:51.766Z{"msg":"[1]R generation[0..12]: [5, 2, 4, 4, 3, 1, 5, 5, 4, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.603661718Z","hostname":"The guest has finished waiting for activation with:6
8422023-09-22T23:23:51.769ZWait for a query_work_queue command to finish before sending IO
8432023-09-22T23:23:51.772Zip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8442023-09-22T23:23:51.775Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.603921Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8452023-09-22T23:23:51.778Z{"msg":"[2]R flush_numbers[0..12]: [7, 2, 6, 6, 3, 1, 7, 7, 6, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.6039974Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8462023-09-22T23:23:51.781Z{"msg":"[2]R generation[0..12]: [5, 2, 4, 4, 3, 1, 5, 5, 4, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.604070519Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8472023-09-22T23:23:51.784Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.604124387Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8482023-09-22T23:23:51.787Z{"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.604182891Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8492023-09-22T23:23:51.790Z{"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.604226842Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8502023-09-22T23:23:51.793Z{"msg":"Next flush: 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.604269334Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8512023-09-22T23:23:51.796Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.604322634Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8522023-09-22T23:23:51.799Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.604384055Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8532023-09-22T23:23:51.802Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.604426167Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8542023-09-22T23:23:51.805Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.604465363Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8552023-09-22T23:23:51.808Z{"msg":"0aee7db3-f87e-48bd-8427-901f3b745d1e is now active with session: d1c80d48-e95f-4486-82fc-5697ca19e314","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.604522252Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8562023-09-22T23:23:51.811Z{"msg":"0aee7db3-f87e-48bd-8427-901f3b745d1e Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.604570573Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8572023-09-22T23:23:51.814Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.604611288Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8582023-09-22T23:23:51.817Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.604673613Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8592023-09-22T23:23:51.820Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
8602023-09-22T23:23:51.822Z,"time":"2023-09-22T23:23:50.604735653Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8612023-09-22T23:23:51.826Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.604832186Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8622023-09-22T23:23:51.829Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.604876844Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8632023-09-22T23:23:51.831Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.604915555Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8642023-09-22T23:23:51.834Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.604953838Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8652023-09-22T23:23:51.837Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.604992159Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8662023-09-22T23:23:51.840Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.6050308Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8672023-09-22T23:23:51.843Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.605069436Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8682023-09-22T23:23:51.846Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.60510668Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1180}
8692023-09-22T23:23:51.849ZLoading write count information from file "/tmp/test_repair_verify.data"
8702023-09-22T23:23:51.852ZRun Repair workload
8712023-09-22T23:23:51.854Z01/30 Write block 402 len 40960 data: 2 2 2 2 2 2 2 3 3 3
8722023-09-22T23:23:51.857Z02/30 Read block 324 len 36864
8732023-09-22T23:23:51.860Z03/30 Read block 337 len 4096
8742023-09-22T23:23:51.863Z04/30 Read block 122 len 40960
8752023-09-22T23:23:51.866Z05/30 Read block 320 len 32768
8762023-09-22T23:23:51.868Z06/30 Flush
8772023-09-22T23:23:51.872Z07/30 Write block 67 len 24576 data: 3 3 3 3 3 3
8782023-09-22T23:23:51.874Z08/30 Read block 78 len 16384
8792023-09-22T23:23:51.877Z09/30 Read block 81 len 20480
8802023-09-22T23:23:51.880Z10/30 Write block 81 len 32768 data: 3 3 3 3 3 3 3 2
8812023-09-22T23:23:51.883Z11/30 Read block 580 len 24576
8822023-09-22T23:23:51.886Z12/30 Read block 46 len 12288
8832023-09-22T23:23:51.888Z13/30 Write block 387 len 4096 data: 3
8842023-09-22T23:23:51.891Z14/30 Flush
8852023-09-22T23:23:51.894Z15/30 Write block 314 len 16384 data: 2 2 2 2
8862023-09-22T23:23:51.897Z16/30 Write block 202 len 28672 data: 2 2 2 2 2 2 2
8872023-09-22T23:23:51.900Z17/30 Read block 214 len 20480
8882023-09-22T23:23:51.902Z18/30 Read block 177 len 40960
8892023-09-22T23:23:51.905Z19/30 Flush
8902023-09-22T23:23:51.908Z20/30 Write block 243 len 4096 data: 2
8912023-09-22T23:23:51.911Z21/30 Read block 239 len 8192
8922023-09-22T23:23:51.914Z22/30 Write block 230 len 4096 data: 2
8932023-09-22T23:23:51.917Z23/30 Read block 207 len 40960
8942023-09-22T23:23:51.920Z24/30 Write block 320 len 4096 data: 2
8952023-09-22T23:23:51.923Z25/30 Write block 531 len 40960 data: 2 2 2 2 2 2 2 2 2 2
8962023-09-22T23:23:51.926Z26/30 Write block 205 len 36864 data: 3 3 3 3 2 2 2 2 2
8972023-09-22T23:23:51.929Z27/30 Read block 206 len 4096
8982023-09-22T23:23:51.932Z28/30 Write block 449 len 28672 data: 2 3 2 2 2 2 2
8992023-09-22T23:23:51.935Z29/30 Write block 87 len 20480 data: 4 3 2 2 2
9002023-09-22T23:23:51.938Z30/30 Read block 224 len 24576
9012023-09-22T23:23:51.941Z----------------------------------------------------------------
9022023-09-22T23:23:51.943Z Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:24
9032023-09-22T23:23:51.947ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
9042023-09-22T23:23:51.961Z 7 Acked 1006 Write 6 Done Done Sent false
9052023-09-22T23:23:51.964Z 8 Acked 1007 Read 4 Done Done Sent false
9062023-09-22T23:23:51.967Z 9 Acked 1008 Read 5 Done Done Sent false
9072023-09-22T23:23:51.970Z 10 Acked 1009 Write 8 Done Done Sent false
9082023-09-22T23:23:51.973Z 11 Acked 1010 Read 6 Done Done Sent false
9092023-09-22T23:23:51.976Z 12 Acked 1011 Read 3 Done Done Sent false
9102023-09-22T23:23:51.979Z 13 Acked 1012 Write 1 Done Done Sent false
9112023-09-22T23:23:51.982Z 14 Acked 1013 Flush 0 Done Done Sent false
9122023-09-22T23:23:51.985Z 15 Acked 1014 Write 4 Done Done Sent false
9132023-09-22T23:23:51.989Z 16 Acked 1015 Write 7 Done Done Sent false
9142023-09-22T23:23:51.992Z 17 Acked 1016 Read 5 Done Done Sent false
9152023-09-22T23:23:51.995Z 18 Acked 1017 Read 10 Done Done Sent false
9162023-09-22T23:23:51.998Z 19 Acked 1018 Flush 0 Done Done Sent false
9172023-09-22T23:23:52.001Z 20 Acked 1019 Write 1 Done Done Sent false
9182023-09-22T23:23:52.004Z 21 Acked 1020 Read 2 Done Done Sent false
9192023-09-22T23:23:52.007Z 22 Acked 1021 Write 1 Done Done Sent false
9202023-09-22T23:23:52.010Z 23 Acked 1022 Read 10 Done Done Sent false
9212023-09-22T23:23:52.013Z 24 Acked 1023 Write 1 Done Done Sent false
9222023-09-22T23:23:52.016Z 25 Acked 1024 Write 10 Done Done Sent false
9232023-09-22T23:23:52.019Z 26 Acked 1025 Write 9 Done Done Sent false
9242023-09-22T23:23:52.022Z 27 Acked 1026 Read 1 Done Done Sent false
9252023-09-22T23:23:52.025Z 28 Acked 1027 Write 7 Done Done Sent false
9262023-09-22T23:23:52.028Z 29 Acked 1028 Write 5 Sent Done Sent false
9272023-09-22T23:23:52.031Z 30 Acked 1029 Read 6 Sent Done Sent false
9282023-09-22T23:23:52.034Z STATES DS:0 DS:1 DS:2 TOTAL
9292023-09-22T23:23:52.037Z New 0 0 0 0
9302023-09-22T23:23:52.041Z Sent 2 0 24 26
9312023-09-22T23:23:52.044Z Done 22 24 0 46
9322023-09-22T23:23:52.047Z Skipped 0 0 0 0
9332023-09-22T23:23:52.050Z Error 0 0 0 0
9342023-09-22T23:23:52.053ZLast Flush: 1018 1018 1005
9352023-09-22T23:23:52.056ZDownstairs last five completed: 1005 1004 1003 1002 1001 1000
9362023-09-22T23:23:52.059ZUpstairs last five completed: 30 29 28 27 26 25
9372023-09-22T23:23:52.062ZWrote out file "/tmp/test_repair_verify.data"
9382023-09-22T23:23:52.065Z
9392023-09-22T23:23:52.067ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
9402023-09-22T23:23:52.070Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.73459656Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1181}
9412023-09-22T23:23:52.073Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.734901404Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1181}
9422023-09-22T23:23:52.076Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.734924626Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1181}
9432023-09-22T23:23:52.078Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.765365259Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1181}
9442023-09-22T23:23:52.081Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.765396963Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1181}
9452023-09-22T23:23:52.084Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.765415518Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1181}
9462023-09-22T23:23:52.087Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.788284882Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1181}
9472023-09-22T23:23:52.089Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.788337662Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1181}
9482023-09-22T23:23:52.092Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:50.788375508Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1181}
9492023-09-22T23:23:52.095ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
9502023-09-22T23:23:52.098Z 0 000-019 5 5 5 7 7 7 F F F
9512023-09-22T23:23:52.101Z 1 020-039 2 2 2 2 2 2 F F F
9522023-09-22T23:23:52.104Z 2 040-059 4 4 4 6 6 6 F F F
9532023-09-22T23:23:52.107Z 3 060-079 6 6 4 9 9 6 F F F <---
9542023-09-22T23:23:52.110Z 4 080-099 6 6 3 9 9 3 T T F <---
9552023-09-22T23:23:52.112Z 5 100-119 1 1 1 1 1 1 F F F
9562023-09-22T23:23:52.115ZError: Difference in extent metadata found!
9572023-09-22T23:23:52.118Z 6 120-139 5 5 5 7 7 7 F F F
9582023-09-22T23:23:52.121Z 7 140-159 5 5 5 7 7 7 F F F
9592023-09-22T23:23:52.123Z 8 160-179 4 4 4 6 6 6 F F F
9602023-09-22T23:23:52.127Z 9 180-199 1 1 1 1 1 1 F F F
9612023-09-22T23:23:52.129Z 10 200-219 6 6 1 10 10 1 T T F <---
9622023-09-22T23:23:52.132Z 11 220-239 1 1 1 1 1 1 T T F <---
9632023-09-22T23:23:52.135Z 12 240-259 3 3 3 3 3 3 T T F <---
9642023-09-22T23:23:52.138Z 13 260-279 3 3 3 3 3 3 F F F
9652023-09-22T23:23:52.141Z 14 280-299 2 2 2 2 2 2 F F F
9662023-09-22T23:23:52.144Z 15 300-319 6 6 1 10 10 1 F F F <---
9672023-09-22T23:23:52.146Z 16 320-339 5 5 5 7 7 7 T T F <---
9682023-09-22T23:23:52.149Z 17 340-359 3 3 3 3 3 3 F F F
9692023-09-22T23:23:52.152Z 18 360-379 3 3 3 3 3 3 F F F
9702023-09-22T23:23:52.155Z 19 380-399 6 6 3 9 9 3 F F F <---
9712023-09-22T23:23:52.157Z 20 400-419 6 6 6 8 8 8 F F F
9722023-09-22T23:23:52.160Z 21 420-439 1 1 1 1 1 1 F F F
9732023-09-22T23:23:52.164Z 22 440-459 4 4 4 6 6 6 T T F <---
9742023-09-22T23:23:52.166Z 23 460-479 5 5 5 7 7 7 F F F
9752023-09-22T23:23:52.169Z 24 480-499 5 5 5 7 7 7 F F F
9762023-09-22T23:23:52.172Z 25 500-519 1 1 1 1 1 1 F F F
9772023-09-22T23:23:52.175Z 26 520-539 1 1 1 1 1 1 T T F <---
9782023-09-22T23:23:52.178Z 27 540-559 5 5 5 7 7 7 T T F <---
9792023-09-22T23:23:52.183Z 28 560-579 1 1 1 1 1 1 F F F
9802023-09-22T23:23:52.185Z 29 580-599 5 5 5 7 7 7 F F F
9812023-09-22T23:23:52.188ZMax gen: 6, Max flush: 10
9822023-09-22T23:23:52.191ZA Difference in extent metadata is expected here
9832023-09-22T23:23:52.193ZOn loop 3
9842023-09-22T23:23:52.196Z
9852023-09-22T23:23:52.199ZVerifying data now
9862023-09-22T23:23:58.180ZNo change after verify
9872023-09-22T23:23:58.183ZLoop: 3 Downstairs dump after verify (and repair):
9882023-09-22T23:23:58.198Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.094943692Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1186}
9892023-09-22T23:23:58.205Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.095268771Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1186}
9902023-09-22T23:23:58.208Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.095294137Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1186}
9912023-09-22T23:23:58.216Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.113958526Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1186}
9922023-09-22T23:23:58.219Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.114014408Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1186}
9932023-09-22T23:23:58.222Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.114035846Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1186}
9942023-09-22T23:23:58.227Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.124775137Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1186}
9952023-09-22T23:23:58.230Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.124827402Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1186}
9962023-09-22T23:23:58.233Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.12484812Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1186}
9972023-09-22T23:23:58.238ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
9982023-09-22T23:23:58.241Z 0 000-019 5 5 5 7 7 7 F F F
9992023-09-22T23:23:58.244Z 1 020-039 2 2 2 2 2 2 F F F
10002023-09-22T23:23:58.247Z 2 040-059 4 4 4 6 6 6 F F F
10012023-09-22T23:23:58.249Z 3 060-079 6 6 6 9 9 9 F F F
10022023-09-22T23:23:58.252Z 4 080-099 7 7 7 11 11 11 F F F
10032023-09-22T23:23:58.255Z 5 100-119 1 1 1 1 1 1 F F F
10042023-09-22T23:23:58.258Z 6 120-139 5 5 5 7 7 7 F F F
10052023-09-22T23:23:58.261Z 7 140-159 5 5 5 7 7 7 F F F
10062023-09-22T23:23:58.263Z 8 160-179 4 4 4 6 6 6 F F F
10072023-09-22T23:23:58.266Z 9 180-199 1 1 1 1 1 1 F F F
10082023-09-22T23:23:58.269Z 10 200-219 7 7 7 11 11 11 F F F
10092023-09-22T23:23:58.277Z 11 220-239 7 7 7 11 11 11 F F F
10102023-09-22T23:23:58.284Z 12 240-259 7 7 7 11 11 11 F F F
10112023-09-22T23:23:58.287Z 13 260-279 3 3 3 3 3 3 F F F
10122023-09-22T23:23:58.289Z 14 280-299 2 2 2 2 2 2 F F F
10132023-09-22T23:23:58.292Z 15 300-319 6 6 6 10 10 10 F F F
10142023-09-22T23:23:58.295Z 16 320-339 7 7 7 11 11 11 F F F
10152023-09-22T23:23:58.298Z 17 340-359 3 3 3 3 3 3 F F F
10162023-09-22T23:23:58.301Z 18 360-379 3 3 3 3 3 3 F F F
10172023-09-22T23:23:58.304Z 19 380-399 6 6 6 9 9 9 F F F
10182023-09-22T23:23:58.308Z 20 400-419 6 6 6 8 8 8 F F F
10192023-09-22T23:23:58.311Z 21 420-439 1 1 1 1 1 1 F F F
10202023-09-22T23:23:58.314Z 22 440-459 7 7 7 11 11 11 F F F
10212023-09-22T23:23:58.317Z 23 460-479 5 5 5 7 7 7 F F F
10222023-09-22T23:23:58.320Z 24 480-499 5 5 5 7 7 7 F F F
10232023-09-22T23:23:58.323Z 25 500-519 1 1 1 1 1 1 F F F
10242023-09-22T23:23:58.325Z 26 520-539 7 7 7 11 11 11 F F F
10252023-09-22T23:23:58.328Z 27 540-559 7 7 7 11 11 11 F F F
10262023-09-22T23:23:58.331Z 28 560-579 1 1 1 1 1 1 F F F
10272023-09-22T23:23:58.333Z 29 580-599 5 5 5 7 7 7 F F F
10282023-09-22T23:23:58.336ZMax gen: 7, Max flush: 11
10292023-09-22T23:23:58.339Z
10302023-09-22T23:23:58.341ZBegin loop 4
10312023-09-22T23:23:58.344ZDownstairs to restart: 0
10322023-09-22T23:23:58.348Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.171517982Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10332023-09-22T23:23:58.351Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.171915852Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10342023-09-22T23:23:58.353Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.171982809Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10352023-09-22T23:23:58.356Z{"msg":"Crucible stats registered with UUID: 3fd3968a-7c9b-4196-8213-866f54b1026f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.172101662Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10362023-09-22T23:23:58.359Z{"msg":"Crucible 3fd3968a-7c9b-4196-8213-866f54b1026f has session id: c80126a8-d91f-4c2b-ba24-658c871eb2c1","v":0,"name":"crucible","level":30Crucible runtime is spawned
10372023-09-22T23:23:58.362Z,"time":"2023-09-22T23:23:57.172144542Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10382023-09-22T23:23:58.364Z{"msg":"up_listen starts"The guest has requested activation with gen:8
10392023-09-22T23:23:58.367Z,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.172377062Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188,"task":"up_listen"}
10402023-09-22T23:23:58.370Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.172434374Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10412023-09-22T23:23:58.372Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.172475389Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10422023-09-22T23:23:58.375Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.172522013Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188,"looper":"1"}
10432023-09-22T23:23:58.378Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.172771842Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188,"looper":"0"}
10442023-09-22T23:23:58.381Z{"msg":"[1] 3fd3968a-7c9b-4196-8213-866f54b1026f looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.172889693Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188,"looper":"1"}
10452023-09-22T23:23:58.384Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.173014644Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188,"looper":"2"}
10462023-09-22T23:23:58.387Z{"msg":"[2] 3fd3968a-7c9b-4196-8213-866f54b1026f looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.17316964Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188,"looper":"2"}
10472023-09-22T23:23:58.389Z{"msg":"3fd3968a-7c9b-4196-8213-866f54b1026f active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.173406448Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10482023-09-22T23:23:58.392Z{"msg":"Set desired generation to :8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.173465996Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10492023-09-22T23:23:58.395Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.173587832Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10502023-09-22T23:23:58.397Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.173896074Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10512023-09-22T23:23:58.400Z{"msg":"[1] 3fd3968a-7c9b-4196-8213-866f54b1026f (dd574b70-7dbd-4915-939c-9de4434ed8ce) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.174205649Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10522023-09-22T23:23:58.403Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.174265611Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10532023-09-22T23:23:58.406Z{"msg":"[1] client is_active_req TRUE, promote! session dd574b70-7dbd-4915-939c-9de4434ed8ce","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.174308732Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10542023-09-22T23:23:58.409Z{"msg":"[2] 3fd3968a-7c9b-4196-8213-866f54b1026f (dd574b70-7dbd-4915-939c-9de4434ed8ce) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.174342975Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10552023-09-22T23:23:58.412Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.17441503Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10562023-09-22T23:23:58.414Z{"msg":"[2] client is_active_req TRUE, promote! session dd574b70-7dbd-4915-939c-9de4434ed8ce","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.174486794Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10572023-09-22T23:23:58.417Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.175033795Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10582023-09-22T23:23:58.420Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.175099113Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10592023-09-22T23:23:58.423Z{"msg":"3fd3968a-7c9b-4196-8213-866f54b1026f New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.175142114Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10602023-09-22T23:23:58.426Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.175274647Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10612023-09-22T23:23:58.428Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.17533034Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10622023-09-22T23:23:58.431Z{"msg":"3fd3968a-7c9b-4196-8213-866f54b1026f New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.175354363Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10632023-09-22T23:23:58.434Z{"msg":"[1] 3fd3968a-7c9b-4196-8213-866f54b1026f (dd574b70-7dbd-4915-939c-9de4434ed8ce) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.177908306Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10642023-09-22T23:23:58.437Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.177980942Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10652023-09-22T23:23:58.440Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:23:57.178089263Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10662023-09-22T23:23:58.442Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.178149208Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10672023-09-22T23:23:58.445Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.178208161Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10682023-09-22T23:23:58.449Z{"msg":"3fd3968a-7c9b-4196-8213-866f54b1026f New WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.178263944Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10692023-09-22T23:23:58.452Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.178317386Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10702023-09-22T23:23:58.455Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.178370461Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10712023-09-22T23:23:58.457Z{"msg":"[2] 3fd3968a-7c9b-4196-8213-866f54b1026f (dd574b70-7dbd-4915-939c-9de4434ed8ce) New WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.178427838Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10722023-09-22T23:23:58.460Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.178481332Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10732023-09-22T23:23:58.463Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:23:57.178534067Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10742023-09-22T23:23:58.465Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.178586515Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10752023-09-22T23:23:58.468Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.178639506Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10762023-09-22T23:23:58.471Z{"msg":"3fd3968a-7c9b-4196-8213-866f54b1026f New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.178669093Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10772023-09-22T23:23:58.473Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.178688548Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10782023-09-22T23:23:58.476Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:57.178705638Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10792023-09-22T23:23:59.278Z{"msg":"[0] 3fd3968a-7c9b-4196-8213-866f54b1026f looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.174272863Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188,"looper":"0"}
10802023-09-22T23:23:59.281Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.174357668Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10812023-09-22T23:23:59.284Z{"msg":"[0] 3fd3968a-7c9b-4196-8213-866f54b1026f (dd574b70-7dbd-4915-939c-9de4434ed8ce) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.175314892Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10822023-09-22T23:23:59.287Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.175349384Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10832023-09-22T23:23:59.290Z{"msg":"[0] client is_active_req TRUE, promote! session dd574b70-7dbd-4915-939c-9de4434ed8ce","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.175408027Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10842023-09-22T23:23:59.293Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.176043428Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10852023-09-22T23:23:59.296Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.1760997Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10862023-09-22T23:23:59.299Z{"msg":"3fd3968a-7c9b-4196-8213-866f54b1026f WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.176153553Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10872023-09-22T23:23:59.302Z{"msg":"[0] 3fd3968a-7c9b-4196-8213-866f54b1026f (dd574b70-7dbd-4915-939c-9de4434ed8ce) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.17843365Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10882023-09-22T23:23:59.305Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.178510705Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10892023-09-22T23:23:59.309Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:23:58.178584666Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10902023-09-22T23:23:59.312Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.178656053Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10912023-09-22T23:23:59.314Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.178709062Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10922023-09-22T23:23:59.317Z{"msg":"3fd3968a-7c9b-4196-8213-866f54b1026f WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.178750391Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10932023-09-22T23:23:59.320Z{"msg":"[0]R flush_numbers[0..12]: [7, 2, 6, 9, 11, 1, 7, 7, 6, 1, 11, 11]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.178797102Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10942023-09-22T23:23:59.323Z{"msg":"[0]R generation[0..12]: [5, 2, 4, 6, 7, 1, 5, 5, 4, 1, 7, 7]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.178836203Z","The guest has finished waiting for activation with:8
10952023-09-22T23:23:59.326Zhostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10962023-09-22T23:23:59.329ZWait for a query_work_queue command to finish before sending IO
10972023-09-22T23:23:59.332Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.178866983Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10982023-09-22T23:23:59.335Z{"msg":"[1]R flush_numbers[0..12]: [7, 2, 6, 9, 11, 1, 7, 7, 6, 1, 11, 11]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.178910357Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
10992023-09-22T23:23:59.338Z{"msg":"[1]R generation[0..12]: [5, 2, 4, 6, 7, 1, 5, 5, 4, 1, 7, 7]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.178971602Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
11002023-09-22T23:23:59.341Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.179051248Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
11012023-09-22T23:23:59.344Z{"msg":"[2]R flush_numbers[0..12]: [7, 2, 6, 9, 11, 1, 7, 7, 6, 1, 11, 11]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.179112939Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
11022023-09-22T23:23:59.348Z{"msg":"[2]R generation[0..12]: [5, 2, 4, 6, 7, 1, 5, 5, 4, 1, 7, 7]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.179182384Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
11032023-09-22T23:23:59.351Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.179253878Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
11042023-09-22T23:23:59.355Z{"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.179324784Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
11052023-09-22T23:23:59.357Z{"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.179392191Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
11062023-09-22T23:23:59.361Z{"msg":"Next flush: 12","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.17946065Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
11072023-09-22T23:23:59.364Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.179535118Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
11082023-09-22T23:23:59.366Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.179605356Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
11092023-09-22T23:23:59.370Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
11102023-09-22T23:23:59.373Z,"time":"2023-09-22T23:23:58.179682994Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
11112023-09-22T23:23:59.376Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.179793979Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
11122023-09-22T23:23:59.378Z{"msg":"3fd3968a-7c9b-4196-8213-866f54b1026f is now active with session: dd574b70-7dbd-4915-939c-9de4434ed8ce","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.179843017Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
11132023-09-22T23:23:59.381Z{"msg":"3fd3968a-7c9b-4196-8213-866f54b1026f Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.179882303Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
11142023-09-22T23:23:59.384Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.17992093Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
11152023-09-22T23:23:59.387Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.179958529Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
11162023-09-22T23:23:59.390Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.180009747Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
11172023-09-22T23:23:59.393Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.180052756Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
11182023-09-22T23:23:59.396Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.180091371Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
11192023-09-22T23:23:59.399Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.180128968Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
11202023-09-22T23:23:59.402Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.180166167Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
11212023-09-22T23:23:59.404Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.18020353Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
11222023-09-22T23:23:59.407Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.180240833Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
11232023-09-22T23:23:59.410ZLoading write count information from file "/tmp/test_repair_verify.data"
11242023-09-22T23:23:59.413Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.180283936Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
11252023-09-22T23:23:59.416ZRun Repair workload
11262023-09-22T23:23:59.418Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.180305899Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1188}
11272023-09-22T23:23:59.421Z01/30 Write block 68 len 24576 data: 4 4 4 4 4 2
11282023-09-22T23:23:59.424Z02/30 Write block 458 len 24576 data: 2 2 2 3 2 2
11292023-09-22T23:23:59.428Z03/30 Read block 344 len 40960
11302023-09-22T23:23:59.431Z04/30 Read block 176 len 4096
11312023-09-22T23:23:59.434Z05/30 Read block 449 len 36864
11322023-09-22T23:23:59.437Z06/30 Read block 552 len 32768
11332023-09-22T23:23:59.440Z07/30 Flush
11342023-09-22T23:23:59.443Z08/30 Write block 95 len 20480 data: 2 2 2 2 2
11352023-09-22T23:23:59.446Z09/30 Read block 494 len 40960
11362023-09-22T23:23:59.449Z10/30 Write block 441 len 36864 data: 2 2 2 2 2 2 2 2 3
11372023-09-22T23:23:59.452Z11/30 Write block 334 len 24576 data: 2 2 2 2 2 2
11382023-09-22T23:23:59.455Z12/30 Read block 358 len 24576
11392023-09-22T23:23:59.457Z13/30 Write block 179 len 16384 data: 2 2 2 2
11402023-09-22T23:23:59.460Z14/30 Write block 297 len 16384 data: 2 2 2 2
11412023-09-22T23:23:59.463Z15/30 Write block 92 len 28672 data: 2 2 2 3 3 3 3
11422023-09-22T23:23:59.468Z16/30 Read block 503 len 4096
11432023-09-22T23:23:59.472Z17/30 Read block 510 len 32768
11442023-09-22T23:23:59.475Z18/30 Read block 467 len 40960
11452023-09-22T23:23:59.478Z19/30 Read block 409 len 32768
11462023-09-22T23:23:59.481Z20/30 Read block 314 len 20480
11472023-09-22T23:23:59.483Z21/30 Write block 282 len 40960 data: 2 2 2 2 2 2 2 2 2 2
11482023-09-22T23:23:59.486Z22/30 Write block 132 len 8192 data: 3 2
11492023-09-22T23:23:59.489Z23/30 Write block 42 len 8192 data: 2 3
11502023-09-22T23:23:59.492Z24/30 Flush
11512023-09-22T23:23:59.494Z25/30 Write block 81 len 40960 data: 4 4 4 4 4 4 5 4 3 3
11522023-09-22T23:23:59.498Z26/30 Read block 342 len 16384
11532023-09-22T23:23:59.501Z27/30 Read block 259 len 24576
11542023-09-22T23:23:59.504Z28/30 Write block 103 len 40960 data: 2 2 2 2 2 2 2 2 2 2
11552023-09-22T23:23:59.506Z29/30 Write block 593 len 28672 data: 2 2 2 2 2 2 2
11562023-09-22T23:23:59.509Z30/30 Write block 427 len 24576 data: 2 2 2 2 2 2
11572023-09-22T23:23:59.512Z----------------------------------------------------------------
11582023-09-22T23:23:59.515Z Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:30
11592023-09-22T23:23:59.518ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
11602023-09-22T23:23:59.521Z 1 Acked 1000 Write 6 Done Done Done false
11612023-09-22T23:23:59.523Z 2 Acked 1001 Write 6 Sent Done Done false
11622023-09-22T23:23:59.526Z 3 Acked 1002 Read 10 Sent Done Done false
11632023-09-22T23:23:59.530Z 4 Acked 1003 Read 1 Sent Done Done false
11642023-09-22T23:23:59.532Z 5 Acked 1004 Read 9 Sent Done Done false
11652023-09-22T23:23:59.535Z 6 Acked 1005 Read 8 Sent Done Done false
11662023-09-22T23:23:59.538Z 7 Acked 1006 Flush 0 Sent Done Done false
11672023-09-22T23:23:59.541Z 8 Acked 1007 Write 5 Sent Done Done false
11682023-09-22T23:23:59.543Z 9 Acked 1008 Read 10 Sent Done Done false
11692023-09-22T23:23:59.546Z 10 Acked 1009 Write 9 Sent Done Done false
11702023-09-22T23:23:59.549Z 11 Acked 1010 Write 6 Sent Done Done false
11712023-09-22T23:23:59.552Z 12 Acked 1011 Read 6 Sent Done Done false
11722023-09-22T23:23:59.555Z 13 Acked 1012 Write 4 Sent Done Done false
11732023-09-22T23:23:59.557Z 14 Acked 1013 Write 4 Sent Done Done false
11742023-09-22T23:23:59.560Z 15 Acked 1014 Write 7 Sent Done Done false
11752023-09-22T23:23:59.563Z 16 Acked 1015 Read 1 Sent Done Done false
11762023-09-22T23:23:59.566Z 17 Acked 1016 Read 8 Sent Done Done false
11772023-09-22T23:23:59.568Z 18 Acked 1017 Read 10 Sent Done Done false
11782023-09-22T23:23:59.571Z 19 Acked 1018 Read 8 Sent Done Done false
11792023-09-22T23:23:59.574Z 20 Acked 1019 Read 5 Sent Done Done false
11802023-09-22T23:23:59.577Z 21 Acked 1020 Write 10 Sent Done Done false
11812023-09-22T23:23:59.580Z 22 Acked 1021 Write 2 Sent Done Done false
11822023-09-22T23:23:59.582Z 23 Acked 1022 Write 2 Sent Done Done false
11832023-09-22T23:23:59.589Z 24 Acked 1023 Flush 0 Sent Done Done false
11842023-09-22T23:23:59.592Z 25 Acked 1024 Write 10 Sent Done Done false
11852023-09-22T23:23:59.594Z 26 Acked 1025 Read 4 Sent Done Done false
11862023-09-22T23:23:59.597Z 27 Acked 1026 Read 6 Sent Done Done false
11872023-09-22T23:23:59.600Z 28 Acked 1027 Write 10 Sent Sent Sent false
11882023-09-22T23:23:59.603Z 29 Acked 1028 Write 7 Sent Sent Sent false
11892023-09-22T23:23:59.606Z 30 Acked 1029 Write 6 Sent Sent Sent false
11902023-09-22T23:23:59.609Z STATES DS:0 DS:1 DS:2 TOTAL
11912023-09-22T23:23:59.611Z New 0 0 0 0
11922023-09-22T23:23:59.614Z Sent 29 3 3 35
11932023-09-22T23:23:59.617Z Done 1 27 27 55
11942023-09-22T23:23:59.619Z Skipped 0 0 0 0
11952023-09-22T23:23:59.622Z Error 0 0 0 0
11962023-09-22T23:23:59.625ZLast Flush: 0 1023 1023
11972023-09-22T23:23:59.628ZDownstairs last five completed:
11982023-09-22T23:23:59.631ZUpstairs last five completed: 30 29 28 27 26 25
11992023-09-22T23:23:59.634ZWrote out file "/tmp/test_repair_verify.data"
12002023-09-22T23:23:59.637Z
12012023-09-22T23:23:59.640ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
12022023-09-22T23:23:59.643Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.319082585Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1189}
12032023-09-22T23:23:59.647Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.319336235Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1189}
12042023-09-22T23:23:59.650Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.319359702Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1189}
12052023-09-22T23:23:59.653Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.359669157Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1189}
12062023-09-22T23:23:59.660Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.359725123Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1189}
12072023-09-22T23:23:59.663Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.359747246Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1189}
12082023-09-22T23:23:59.666Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.377003635Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1189}
12092023-09-22T23:23:59.669Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.377057248Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1189}
12102023-09-22T23:23:59.672Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:23:58.377095904Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1189}
12112023-09-22T23:23:59.674ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12122023-09-22T23:23:59.677Z 0 000-019 5 5 5 7 7 7 F F F
12132023-09-22T23:23:59.681Z 1 020-039 2 2 2 2 2 2 F F F
12142023-09-22T23:23:59.684Z 2 040-059 4 8 8 6 13 13 F F F <---
12152023-09-22T23:23:59.687Z 3 060-079 6 8 8 9 12 12 T F F <---
12162023-09-22T23:23:59.690Z 4 080-099 7 8 8 11 13 13 F T T <---
12172023-09-22T23:23:59.693ZError: Difference in extent metadata found!
12182023-09-22T23:23:59.696Z 5 100-119 1 1 1 1 1 1 F T T <---
12192023-09-22T23:23:59.699Z 6 120-139 5 8 8 7 13 13 F F F <---
12202023-09-22T23:23:59.703Z 7 140-159 5 5 5 7 7 7 F F F
12212023-09-22T23:23:59.706Z 8 160-179 4 8 8 6 13 13 F F F <---
12222023-09-22T23:23:59.709Z 9 180-199 1 8 8 1 13 13 F F F <---
12232023-09-22T23:23:59.712Z 10 200-219 7 7 7 11 11 11 F F F
12242023-09-22T23:23:59.715Z 11 220-239 7 7 7 11 11 11 F F F
12252023-09-22T23:23:59.718Z 12 240-259 7 7 7 11 11 11 F F F
12262023-09-22T23:23:59.720Z 13 260-279 3 3 3 3 3 3 F F F
12272023-09-22T23:23:59.723Z 14 280-299 2 8 8 2 13 13 F F F <---
12282023-09-22T23:23:59.726Z 15 300-319 6 8 8 10 13 13 F F F <---
12292023-09-22T23:23:59.729Z 16 320-339 7 8 8 11 13 13 F F F <---
12302023-09-22T23:23:59.733Z 17 340-359 3 3 3 3 3 3 F F F
12312023-09-22T23:23:59.736Z 18 360-379 3 3 3 3 3 3 F F F
12322023-09-22T23:23:59.739Z 19 380-399 6 6 6 9 9 9 F F F
12332023-09-22T23:23:59.742Z 20 400-419 6 6 6 8 8 8 F F F
12342023-09-22T23:23:59.745Z 21 420-439 1 1 1 1 1 1 F T F <---
12352023-09-22T23:23:59.748Z 22 440-459 7 8 8 11 13 13 F F F <---
12362023-09-22T23:23:59.751Z 23 460-479 5 8 8 7 12 12 F F F <---
12372023-09-22T23:23:59.754Z 24 480-499 5 5 5 7 7 7 F F F
12382023-09-22T23:23:59.756Z 25 500-519 1 1 1 1 1 1 F F F
12392023-09-22T23:23:59.759Z 26 520-539 7 7 7 11 11 11 F F F
12402023-09-22T23:23:59.762Z 27 540-559 7 7 7 11 11 11 F F F
12412023-09-22T23:23:59.765Z 28 560-579 1 1 1 1 1 1 F F F
12422023-09-22T23:23:59.767Z 29 580-599 5 5 5 7 7 7 F T T <---
12432023-09-22T23:23:59.770ZMax gen: 8, Max flush: 13
12442023-09-22T23:23:59.773ZA Difference in extent metadata is expected here
12452023-09-22T23:23:59.776ZOn loop 4
12462023-09-22T23:23:59.779Z
12472023-09-22T23:23:59.782ZVerifying data now
12482023-09-22T23:24:06.006ZNo change after verify
12492023-09-22T23:24:06.009ZLoop: 4 Downstairs dump after verify (and repair):
12502023-09-22T23:24:06.025Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:04.921609916Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1194}
12512023-09-22T23:24:06.028Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:04.922010213Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1194}
12522023-09-22T23:24:06.031Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:04.922034852Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1194}
12532023-09-22T23:24:06.044Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:04.941060208Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1194}
12542023-09-22T23:24:06.047Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:04.941113533Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1194}
12552023-09-22T23:24:06.050Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:04.941134287Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1194}
12562023-09-22T23:24:06.055Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:04.952185526Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1194}
12572023-09-22T23:24:06.058Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:04.952237249Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1194}
12582023-09-22T23:24:06.061Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:04.952257667Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1194}
12592023-09-22T23:24:06.066ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
12602023-09-22T23:24:06.069Z 0 000-019 5 5 5 7 7 7 F F F
12612023-09-22T23:24:06.072Z 1 020-039 2 2 2 2 2 2 F F F
12622023-09-22T23:24:06.075Z 2 040-059 8 8 8 13 13 13 F F F
12632023-09-22T23:24:06.079Z 3 060-079 8 8 8 12 12 12 F F F
12642023-09-22T23:24:06.082Z 4 080-099 9 9 9 14 14 14 F F F
12652023-09-22T23:24:06.085Z 5 100-119 9 9 9 14 14 14 F F F
12662023-09-22T23:24:06.088Z 6 120-139 8 8 8 13 13 13 F F F
12672023-09-22T23:24:06.090Z 7 140-159 5 5 5 7 7 7 F F F
12682023-09-22T23:24:06.093Z 8 160-179 8 8 8 13 13 13 F F F
12692023-09-22T23:24:06.096Z 9 180-199 8 8 8 13 13 13 F F F
12702023-09-22T23:24:06.099Z 10 200-219 7 7 7 11 11 11 F F F
12712023-09-22T23:24:06.102Z 11 220-239 7 7 7 11 11 11 F F F
12722023-09-22T23:24:06.105Z 12 240-259 7 7 7 11 11 11 F F F
12732023-09-22T23:24:06.108Z 13 260-279 3 3 3 3 3 3 F F F
12742023-09-22T23:24:06.111Z 14 280-299 8 8 8 13 13 13 F F F
12752023-09-22T23:24:06.114Z 15 300-319 8 8 8 13 13 13 F F F
12762023-09-22T23:24:06.117Z 16 320-339 8 8 8 13 13 13 F F F
12772023-09-22T23:24:06.120Z 17 340-359 3 3 3 3 3 3 F F F
12782023-09-22T23:24:06.122Z 18 360-379 3 3 3 3 3 3 F F F
12792023-09-22T23:24:06.125Z 19 380-399 6 6 6 9 9 9 F F F
12802023-09-22T23:24:06.128Z 20 400-419 6 6 6 8 8 8 F F F
12812023-09-22T23:24:06.131Z 21 420-439 9 9 9 14 14 14 F F F
12822023-09-22T23:24:06.133Z 22 440-459 8 8 8 13 13 13 F F F
12832023-09-22T23:24:06.136Z 23 460-479 8 8 8 12 12 12 F F F
12842023-09-22T23:24:06.139Z 24 480-499 5 5 5 7 7 7 F F F
12852023-09-22T23:24:06.142Z 25 500-519 1 1 1 1 1 1 F F F
12862023-09-22T23:24:06.144Z 26 520-539 7 7 7 11 11 11 F F F
12872023-09-22T23:24:06.147Z 27 540-559 7 7 7 11 11 11 F F F
12882023-09-22T23:24:06.150Z 28 560-579 1 1 1 1 1 1 F F F
12892023-09-22T23:24:06.153Z 29 580-599 9 9 9 14 14 14 F F F
12902023-09-22T23:24:06.156ZMax gen: 9, Max flush: 14
12912023-09-22T23:24:06.158Z
12922023-09-22T23:24:06.161ZBegin loop 5
12932023-09-22T23:24:06.164ZDownstairs to restart: 1
12942023-09-22T23:24:06.167Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.001341655Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
12952023-09-22T23:24:06.170Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.001721303Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
12962023-09-22T23:24:06.172Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.001776777Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
12972023-09-22T23:24:06.175Z{"msg":"Crucible stats registered with UUID: 90b3791c-ae97-42dd-b502-071968da2692","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.00182052Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
12982023-09-22T23:24:06.179Z{"msg":"Crucible 90b3791c-ae97-42dd-b502-071968da2692 has session id: 9a078699-f617-4105-965d-3f82a9e49f29","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.001880664Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
12992023-09-22T23:24:06.182ZCrucible runtime is spawned
13002023-09-22T23:24:06.184ZThe guest has requested activation with gen:10
13012023-09-22T23:24:06.188Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.002121957Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196,"task":"up_listen"}
13022023-09-22T23:24:06.191Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.002241414Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13032023-09-22T23:24:06.194Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.0023065Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13042023-09-22T23:24:06.197Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.002351089Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196,"looper":"1"}
13052023-09-22T23:24:06.199Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.002571733Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196,"looper":"2"}
13062023-09-22T23:24:06.202Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.002792063Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196,"looper":"0"}
13072023-09-22T23:24:06.205Z{"msg":"[2] 90b3791c-ae97-42dd-b502-071968da2692 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.002876165Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196,"looper":"2"}
13082023-09-22T23:24:06.210Z{"msg":"[0] 90b3791c-ae97-42dd-b502-071968da2692 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.002950736Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196,"looper":"0"}
13092023-09-22T23:24:06.213Z{"msg":"90b3791c-ae97-42dd-b502-071968da2692 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.003211004Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13102023-09-22T23:24:06.216Z{"msg":"Set desired generation to :10","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.003264287Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13112023-09-22T23:24:06.219Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.003395844Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13122023-09-22T23:24:06.222Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.003710754Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13132023-09-22T23:24:06.225Z{"msg":"[2] 90b3791c-ae97-42dd-b502-071968da2692 (09c2397f-01a9-422d-a6e8-71f92658c52d) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.004022412Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13142023-09-22T23:24:06.228Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.004084997Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13152023-09-22T23:24:06.231Z{"msg":"[2] client is_active_req TRUE, promote! session 09c2397f-01a9-422d-a6e8-71f92658c52d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.004128743Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13162023-09-22T23:24:06.233Z{"msg":"[0] 90b3791c-ae97-42dd-b502-071968da2692 (09c2397f-01a9-422d-a6e8-71f92658c52d) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.0041625Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13172023-09-22T23:24:06.236Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.00426686Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13182023-09-22T23:24:06.239Z{"msg":"[0] client is_active_req TRUE, promote! session 09c2397f-01a9-422d-a6e8-71f92658c52d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.00432023Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13192023-09-22T23:24:06.242Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.004749417Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13202023-09-22T23:24:06.245Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.004813255Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13212023-09-22T23:24:06.248Z{"msg":"90b3791c-ae97-42dd-b502-071968da2692 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.004915468Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13222023-09-22T23:24:06.253Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.004963272Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13232023-09-22T23:24:06.256Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.004990209Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13242023-09-22T23:24:06.258Z{"msg":"90b3791c-ae97-42dd-b502-071968da2692 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.005029004Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13252023-09-22T23:24:06.261Z{"msg":"[2] 90b3791c-ae97-42dd-b502-071968da2692 (09c2397f-01a9-422d-a6e8-71f92658c52d) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.007075743Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13262023-09-22T23:24:06.264Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.007160664Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13272023-09-22T23:24:06.267Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:05.007220473Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13282023-09-22T23:24:06.270Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.007264122Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13292023-09-22T23:24:06.272Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.007303995Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13302023-09-22T23:24:06.275Z{"msg":"90b3791c-ae97-42dd-b502-071968da2692 WaitActive New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.007343773Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13312023-09-22T23:24:06.278Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.007382767Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13322023-09-22T23:24:06.281Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.007421375Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13332023-09-22T23:24:06.283Z{"msg":"[0] 90b3791c-ae97-42dd-b502-071968da2692 (09c2397f-01a9-422d-a6e8-71f92658c52d) WaitActive New WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.007463032Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13342023-09-22T23:24:06.286Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.007502717Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13352023-09-22T23:24:06.289Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:05.007541856Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13362023-09-22T23:24:06.292Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.007580645Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13372023-09-22T23:24:06.295Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.007619551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13382023-09-22T23:24:06.298Z{"msg":"90b3791c-ae97-42dd-b502-071968da2692 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.007658501Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13392023-09-22T23:24:06.300Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.007678988Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13402023-09-22T23:24:06.303Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:05.007696329Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13412023-09-22T23:24:07.107Z{"msg":"[1] 90b3791c-ae97-42dd-b502-071968da2692 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.00340136Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196,"looper":"1"}
13422023-09-22T23:24:07.110Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.003498208Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13432023-09-22T23:24:07.113Z{"msg":"[1] 90b3791c-ae97-42dd-b502-071968da2692 (09c2397f-01a9-422d-a6e8-71f92658c52d) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.004717188Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13442023-09-22T23:24:07.116Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.004776492Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13452023-09-22T23:24:07.119Z{"msg":"[1] client is_active_req TRUE, promote! session 09c2397f-01a9-422d-a6e8-71f92658c52d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.00480276Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13462023-09-22T23:24:07.123Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.005414231Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13472023-09-22T23:24:07.126Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.005474522Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13482023-09-22T23:24:07.129Z{"msg":"90b3791c-ae97-42dd-b502-071968da2692 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.005517754Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13492023-09-22T23:24:07.132Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.007927431Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13502023-09-22T23:24:07.135Z{"msg":"[1] 90b3791c-ae97-42dd-b502-071968da2692 (09c2397f-01a9-422d-a6e8-71f92658c52d) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.008068612Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13512023-09-22T23:24:07.138Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.008128151Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13522023-09-22T23:24:07.141Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:06.008170869Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13532023-09-22T23:24:07.144Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.008210274Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13542023-09-22T23:24:07.147Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.008249375Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13552023-09-22T23:24:07.150Z{"msg":"90b3791c-ae97-42dd-b502-071968da2692 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.008289474Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13562023-09-22T23:24:07.153Z{"msg":"[0]R flush_numbers[0..12]: [7, 2, 13, 12, 14, 14, 13, 7, 13, 13, 11, 11]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.008329479Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13572023-09-22T23:24:07.155Z{"msg":"[0]R generation[0..12]: [5, 2, 8, 8, 9, 9, 8, 5, 8, 8, 7, 7]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.008376538Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13582023-09-22T23:24:07.159Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.008416696Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13592023-09-22T23:24:07.161Z{"msg":"The guest has finished waiting for activation with:10
13602023-09-22T23:24:07.165Z[1]R flush_numbers[0..12]: [7, 2, 13, 12, 14, 14, 13, 7, 13, 13, 11, 11]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
13612023-09-22T23:24:07.167Z,"time":"2023-09-22T23:24:06.008442633Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13622023-09-22T23:24:07.171Z{"msg":"[1]R generation[0..12]: [5, 2, 8, 8, 9, 9, 8, 5, 8, 8, 7, 7]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.008463984Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13632023-09-22T23:24:07.174Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.008519389Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13642023-09-22T23:24:07.176Z{"msg":"[2]R flush_numbers[0..12]: [7, 2, 13, 12, 14, 14, 13, 7, 13, 13, 11, 11]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.008585069Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13652023-09-22T23:24:07.179Z{"msg":"[2]R generation[0..12]: [5, 2, 8, 8, 9, 9, 8, 5, 8, 8, 7, 7]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.008658677Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13662023-09-22T23:24:07.182Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.008704909Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13672023-09-22T23:24:07.185Z{"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.008765894Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13682023-09-22T23:24:07.187Z{"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.008834171Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13692023-09-22T23:24:07.190Z{"msg":"Next flush: 15","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.008900256Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13702023-09-22T23:24:07.194Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.008956224Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13712023-09-22T23:24:07.197Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.009017429Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13722023-09-22T23:24:07.200Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.009073673Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13732023-09-22T23:24:07.202Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.009128439Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13742023-09-22T23:24:07.205Z{"msg":"90b3791c-ae97-42dd-b502-071968da2692 is now active with session: 09c2397f-01a9-422d-a6e8-71f92658c52d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.009186269Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13752023-09-22T23:24:07.208Z{"msg":"90b3791c-ae97-42dd-b502-071968da2692 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.00924584Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13762023-09-22T23:24:07.211Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.009301114Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13772023-09-22T23:24:07.214Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.00935606Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13782023-09-22T23:24:07.216Z{"msg":"[2] received reconcile message","v":0,"name":"crucible"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
13792023-09-22T23:24:07.219Z,"level":30,"time":"2023-09-22T23:24:06.009489579Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13802023-09-22T23:24:07.222Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.009547328Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13812023-09-22T23:24:07.225Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.009601377Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13822023-09-22T23:24:07.228Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.009655653Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13832023-09-22T23:24:07.231Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.009710633Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13842023-09-22T23:24:07.234Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.009763981Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13852023-09-22T23:24:07.237Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.009816566Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13862023-09-22T23:24:07.239Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.009869107Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13872023-09-22T23:24:07.242Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.009922407Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1196}
13882023-09-22T23:24:07.245ZLoading write count information from file "/tmp/test_repair_verify.data"
13892023-09-22T23:24:07.248ZRun Repair workload
13902023-09-22T23:24:07.251Z01/30 Write block 380 len 36864 data: 2 2 2 2 2 2 2 4 3
13912023-09-22T23:24:07.254Z02/30 Write block 34 len 12288 data: 2 2 2
13922023-09-22T23:24:07.257Z03/30 Write block 288 len 36864 data: 3 3 3 3 2 2 2 2 2
13932023-09-22T23:24:07.260Z04/30 Read block 472 len 24576
13942023-09-22T23:24:07.262Z05/30 Read block 55 len 12288
13952023-09-22T23:24:07.265Z06/30 Flush
13962023-09-22T23:24:07.268Z07/30 Flush
13972023-09-22T23:24:07.271Z08/30 Flush
13982023-09-22T23:24:07.274Z09/30 Write block 342 len 20480 data: 2 2 2 2 2
13992023-09-22T23:24:07.276Z10/30 Write block 76 len 28672 data: 2 2 2 3 3 5 5
14002023-09-22T23:24:07.279Z11/30 Read block 281 len 24576
14012023-09-22T23:24:07.282Z12/30 Write block 222 len 20480 data: 2 2 2 2 2
14022023-09-22T23:24:07.285Z13/30 Write block 497 len 40960 data: 2 2 2 2 2 2 2 2 2 2
14032023-09-22T23:24:07.288Z14/30 Read block 577 len 16384
14042023-09-22T23:24:07.291Z15/30 Write block 509 len 28672 data: 2 2 2 2 2 2 2
14052023-09-22T23:24:07.293Z16/30 Write block 189 len 8192 data: 2 2
14062023-09-22T23:24:07.296Z17/30 Read block 373 len 12288
14072023-09-22T23:24:07.299Z18/30 Read block 268 len 24576
14082023-09-22T23:24:07.302Z19/30 Write block 62 len 32768 data: 2 2 2 2 2 4 5 5
14092023-09-22T23:24:07.305Z20/30 Write block 268 len 24576 data: 3 3 3 3 3 3
14102023-09-22T23:24:07.324Z21/30 Read block 230 len 32768
14112023-09-22T23:24:07.327Z22/30 Read block 585 len 32768
14122023-09-22T23:24:07.330Z23/30 Read block 118 len 24576
14132023-09-22T23:24:07.333Z24/30 Write block 366 len 8192 data: 2 2
14142023-09-22T23:24:07.336Z25/30 Read block 366 len 32768
14152023-09-22T23:24:07.339Z26/30 Read block 489 len 12288
14162023-09-22T23:24:07.342Z27/30 Write block 36 len 36864 data: 3 2 2 2 2 2 3 4 3
14172023-09-22T23:24:07.345Z28/30 Write block 347 len 4096 data: 2
14182023-09-22T23:24:07.348Z29/30 Read block 122 len 20480
14192023-09-22T23:24:07.351Z30/30 Write block 206 len 40960 data: 4 4 4 3 3 3 3 3 2 2
14202023-09-22T23:24:07.354Z----------------------------------------------------------------
14212023-09-22T23:24:07.357Z Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30
14222023-09-22T23:24:07.361ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
14232023-09-22T23:24:07.364Z 1 Acked 1000 Write 9 Done Done Done false
14242023-09-22T23:24:07.367Z 2 Acked 1001 Write 3 Done Done Done false
14252023-09-22T23:24:07.370Z 3 Acked 1002 Write 9 Done Done Done false
14262023-09-22T23:24:07.373Z 4 Acked 1003 Read 6 Done Done Done false
14272023-09-22T23:24:07.376Z 5 Acked 1004 Read 3 Done Sent Done false
14282023-09-22T23:24:07.378Z 6 Acked 1005 Flush 0 Done Sent Done false
14292023-09-22T23:24:07.381Z 7 Acked 1006 Flush 0 Done Sent Done false
14302023-09-22T23:24:07.384Z 8 Acked 1007 Flush 0 Done Sent Done false
14312023-09-22T23:24:07.387Z 9 Acked 1008 Write 5 Done Sent Done false
14322023-09-22T23:24:07.390Z 10 Acked 1009 Write 7 Done Sent Done false
14332023-09-22T23:24:07.393Z 11 Acked 1010 Read 6 Done Sent Done false
14342023-09-22T23:24:07.396Z 12 Acked 1011 Write 5 Done Sent Done false
14352023-09-22T23:24:07.401Z 13 Acked 1012 Write 10 Done Sent Done false
14362023-09-22T23:24:07.404Z 14 Acked 1013 Read 4 Done Sent Done false
14372023-09-22T23:24:07.407Z 15 Acked 1014 Write 7 Done Sent Done false
14382023-09-22T23:24:07.410Z 16 Acked 1015 Write 2 Done Sent Done false
14392023-09-22T23:24:07.413Z 17 Acked 1016 Read 3 Done Sent Done false
14402023-09-22T23:24:07.416Z 18 Acked 1017 Read 6 Done Sent Done false
14412023-09-22T23:24:07.419Z 19 Acked 1018 Write 8 Done Sent Done false
14422023-09-22T23:24:07.422Z 20 Acked 1019 Write 6 Done Sent Done false
14432023-09-22T23:24:07.425Z 21 Acked 1020 Read 8 Done Sent Done false
14442023-09-22T23:24:07.428Z 22 Acked 1021 Read 8 Done Sent Done false
14452023-09-22T23:24:07.431Z 23 Acked 1022 Read 6 Done Sent Done false
14462023-09-22T23:24:07.434Z 24 Acked 1023 Write 2 Done Sent Done false
14472023-09-22T23:24:07.437Z 25 Acked 1024 Read 8 Done Sent Done false
14482023-09-22T23:24:07.439Z 26 Acked 1025 Read 3 Done Sent Done false
14492023-09-22T23:24:07.442Z 27 Acked 1026 Write 9 Done Sent Done false
14502023-09-22T23:24:07.445Z 28 Acked 1027 Write 1 Done Sent Done false
14512023-09-22T23:24:07.448Z 29 Acked 1028 Read 5 Done Sent Done false
14522023-09-22T23:24:07.451Z 30 Acked 1029 Write 10 Sent Sent Sent false
14532023-09-22T23:24:07.454Z STATES DS:0 DS:1 DS:2 TOTAL
14542023-09-22T23:24:07.456Z New 0 0 0 0
14552023-09-22T23:24:07.460Z Sent 1 26 1 28
14562023-09-22T23:24:07.462Z Done 29 4 29 62
14572023-09-22T23:24:07.465Z Skipped 0 0 0 0
14582023-09-22T23:24:07.468Z Error 0 0 0 0
14592023-09-22T23:24:07.470ZLast Flush: 1007 0 1007
14602023-09-22T23:24:07.473ZDownstairs last five completed:
14612023-09-22T23:24:07.476ZUpstairs last five completed: 30 29 28 27 26 25
14622023-09-22T23:24:07.479ZWrote out file "/tmp/test_repair_verify.data"
14632023-09-22T23:24:07.481Z
14642023-09-22T23:24:07.484ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
14652023-09-22T23:24:07.487Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.146129609Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1197}
14662023-09-22T23:24:07.490Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.146485099Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1197}
14672023-09-22T23:24:07.493Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.146508758Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1197}
14682023-09-22T23:24:07.496Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.184866524Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1197}
14692023-09-22T23:24:07.498Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.18489982Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1197}
14702023-09-22T23:24:07.501Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.184919049Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1197}
14712023-09-22T23:24:07.504Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.218217276Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1197}
14722023-09-22T23:24:07.507Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.218270473Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1197}
14732023-09-22T23:24:07.510Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:06.218308817Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1197}
14742023-09-22T23:24:07.513ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
14752023-09-22T23:24:07.515Z 0 000-019 5 5 5 7 7 7 F F F
14762023-09-22T23:24:07.518Z 1 020-039 10 2 10 15 2 15 T T T <---
14772023-09-22T23:24:07.521Z 2 040-059 8 8 8 13 13 13 T F T <---
14782023-09-22T23:24:07.524Z 3 060-079 8 8 8 12 12 12 T F T <---
14792023-09-22T23:24:07.527Z 4 080-099 9 9 9 14 14 14 T F T <---
14802023-09-22T23:24:07.530Z 5 100-119 9 9 9 14 14 14 F F F
14812023-09-22T23:24:07.533Z 6 120-139 8 8 8 13 13 13 F F F
14822023-09-22T23:24:07.536Z 7 140-159 5 5 5 7 7 7 F F F
14832023-09-22T23:24:07.539Z 8 160-179 8 8 8 13 13 13 F F F
14842023-09-22T23:24:07.542ZError: Difference in extent metadata found!
14852023-09-22T23:24:07.545Z 9 180-199 8 8 8 13 13 13 T F T <---
14862023-09-22T23:24:07.548Z 10 200-219 7 7 7 11 11 11 T F T <---
14872023-09-22T23:24:07.551Z 11 220-239 7 7 7 11 11 11 T F T <---
14882023-09-22T23:24:07.554Z 12 240-259 7 7 7 11 11 11 F F F
14892023-09-22T23:24:07.557Z 13 260-279 3 3 3 3 3 3 T F T <---
14902023-09-22T23:24:07.559Z 14 280-299 10 8 10 15 13 15 F T F <---
14912023-09-22T23:24:07.563Z 15 300-319 8 8 8 13 13 13 F F F
14922023-09-22T23:24:07.566Z 16 320-339 8 8 8 13 13 13 F F F
14932023-09-22T23:24:07.568Z 17 340-359 3 3 3 3 3 3 T F T <---
14942023-09-22T23:24:07.571Z 18 360-379 3 3 3 3 3 3 T F T <---
14952023-09-22T23:24:07.574Z 19 380-399 10 6 10 15 9 15 F T F <---
14962023-09-22T23:24:07.577Z 20 400-419 6 6 6 8 8 8 F F F
14972023-09-22T23:24:07.580Z 21 420-439 9 9 9 14 14 14 F F F
14982023-09-22T23:24:07.583Z 22 440-459 8 8 8 13 13 13 F F F
14992023-09-22T23:24:07.586Z 23 460-479 8 8 8 12 12 12 F F F
15002023-09-22T23:24:07.589Z 24 480-499 5 5 5 7 7 7 T F T <---
15012023-09-22T23:24:07.593Z 25 500-519 1 1 1 1 1 1 T F T <---
15022023-09-22T23:24:07.596Z 26 520-539 7 7 7 11 11 11 F F F
15032023-09-22T23:24:07.599Z 27 540-559 7 7 7 11 11 11 F F F
15042023-09-22T23:24:07.602Z 28 560-579 1 1 1 1 1 1 F F F
15052023-09-22T23:24:07.605Z 29 580-599 9 9 9 14 14 14 F F F
15062023-09-22T23:24:07.608ZMax gen: 10, Max flush: 15
15072023-09-22T23:24:07.611ZA Difference in extent metadata is expected here
15082023-09-22T23:24:07.614ZOn loop 5
15092023-09-22T23:24:07.616Z
15102023-09-22T23:24:07.619ZVerifying data now
15112023-09-22T23:24:13.886ZNo change after verify
15122023-09-22T23:24:13.889ZLoop: 5 Downstairs dump after verify (and repair):
15132023-09-22T23:24:13.904Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.801497856Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1202}
15142023-09-22T23:24:13.907Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.801902493Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1202}
15152023-09-22T23:24:13.910Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.801927061Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1202}
15162023-09-22T23:24:13.923Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.820732874Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1202}
15172023-09-22T23:24:13.926Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.820784385Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1202}
15182023-09-22T23:24:13.929Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.82080487Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1202}
15192023-09-22T23:24:13.934Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.831893104Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1202}
15202023-09-22T23:24:13.937Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.831943958Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1202}
15212023-09-22T23:24:13.940Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.831964207Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1202}
15222023-09-22T23:24:13.946ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
15232023-09-22T23:24:13.949Z 0 000-019 5 5 5 7 7 7 F F F
15242023-09-22T23:24:13.952Z 1 020-039 11 11 11 16 16 16 F F F
15252023-09-22T23:24:13.955Z 2 040-059 11 11 11 16 16 16 F F F
15262023-09-22T23:24:13.958Z 3 060-079 11 11 11 16 16 16 F F F
15272023-09-22T23:24:13.961Z 4 080-099 11 11 11 16 16 16 F F F
15282023-09-22T23:24:13.964Z 5 100-119 9 9 9 14 14 14 F F F
15292023-09-22T23:24:13.967Z 6 120-139 8 8 8 13 13 13 F F F
15302023-09-22T23:24:13.970Z 7 140-159 5 5 5 7 7 7 F F F
15312023-09-22T23:24:13.972Z 8 160-179 8 8 8 13 13 13 F F F
15322023-09-22T23:24:13.975Z 9 180-199 11 11 11 16 16 16 F F F
15332023-09-22T23:24:13.978Z 10 200-219 11 11 11 16 16 16 F F F
15342023-09-22T23:24:13.981Z 11 220-239 11 11 11 16 16 16 F F F
15352023-09-22T23:24:13.984Z 12 240-259 7 7 7 11 11 11 F F F
15362023-09-22T23:24:13.987Z 13 260-279 11 11 11 16 16 16 F F F
15372023-09-22T23:24:13.990Z 14 280-299 10 10 10 15 15 15 F F F
15382023-09-22T23:24:13.993Z 15 300-319 8 8 8 13 13 13 F F F
15392023-09-22T23:24:13.996Z 16 320-339 8 8 8 13 13 13 F F F
15402023-09-22T23:24:13.999Z 17 340-359 11 11 11 16 16 16 F F F
15412023-09-22T23:24:14.002Z 18 360-379 11 11 11 16 16 16 F F F
15422023-09-22T23:24:14.005Z 19 380-399 10 10 10 15 15 15 F F F
15432023-09-22T23:24:14.008Z 20 400-419 6 6 6 8 8 8 F F F
15442023-09-22T23:24:14.011Z 21 420-439 9 9 9 14 14 14 F F F
15452023-09-22T23:24:14.013Z 22 440-459 8 8 8 13 13 13 F F F
15462023-09-22T23:24:14.016Z 23 460-479 8 8 8 12 12 12 F F F
15472023-09-22T23:24:14.019Z 24 480-499 11 11 11 16 16 16 F F F
15482023-09-22T23:24:14.022Z 25 500-519 11 11 11 16 16 16 F F F
15492023-09-22T23:24:14.025Z 26 520-539 7 7 7 11 11 11 F F F
15502023-09-22T23:24:14.027Z 27 540-559 7 7 7 11 11 11 F F F
15512023-09-22T23:24:14.030Z 28 560-579 1 1 1 1 1 1 F F F
15522023-09-22T23:24:14.033Z 29 580-599 9 9 9 14 14 14 F F F
15532023-09-22T23:24:14.036ZMax gen: 11, Max flush: 16
15542023-09-22T23:24:14.039Z
15552023-09-22T23:24:14.042ZBegin loop 6
15562023-09-22T23:24:14.045ZDownstairs to restart: 1
15572023-09-22T23:24:14.048Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.880449729Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15582023-09-22T23:24:14.051Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.880822323Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15592023-09-22T23:24:14.054Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.880852529Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15602023-09-22T23:24:14.058Z{"msg":"Crucible stats registered with UUID: 1762743a-abac-40c6-9217-879068b302ca","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.880915599Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15612023-09-22T23:24:14.060Z{"msg":"Crucible 1762743a-abac-40c6-9217-879068b302ca has session id: 325a47e7-1ba3-485b-a459-35ab3176f2c7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.880984859Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15622023-09-22T23:24:14.063ZCrucible runtime is spawned
15632023-09-22T23:24:14.066ZThe guest has requested activation with gen:12
15642023-09-22T23:24:14.069Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.881255092Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204,"task":"up_listen"}
15652023-09-22T23:24:14.071Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.881313304Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15662023-09-22T23:24:14.074Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.881354963Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15672023-09-22T23:24:14.077Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.881393321Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204,"looper":"2"}
15682023-09-22T23:24:14.080Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.881645711Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204,"looper":"0"}
15692023-09-22T23:24:14.082Z{"msg":"[2] 1762743a-abac-40c6-9217-879068b302ca looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.881728358Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204,"looper":"2"}
15702023-09-22T23:24:14.085Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.881798691Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204,"looper":"1"}
15712023-09-22T23:24:14.088Z{"msg":"[0] 1762743a-abac-40c6-9217-879068b302ca looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.88185199Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204,"looper":"0"}
15722023-09-22T23:24:14.091Z{"msg":"1762743a-abac-40c6-9217-879068b302ca active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.882220452Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15732023-09-22T23:24:14.113Z{"msg":"Set desired generation to :12","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.882280973Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15742023-09-22T23:24:14.120Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.8823627Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15752023-09-22T23:24:14.123Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.882738006Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15762023-09-22T23:24:14.125Z{"msg":"[2] 1762743a-abac-40c6-9217-879068b302ca (9b5fb6af-ef0d-4296-aeda-f1dea38a8f72) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.883005789Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15772023-09-22T23:24:14.128Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.883062794Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15782023-09-22T23:24:14.131Z{"msg":"[2] client is_active_req TRUE, promote! session 9b5fb6af-ef0d-4296-aeda-f1dea38a8f72","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.883132414Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15792023-09-22T23:24:14.134Z{"msg":"[0] 1762743a-abac-40c6-9217-879068b302ca (9b5fb6af-ef0d-4296-aeda-f1dea38a8f72) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.883210747Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15802023-09-22T23:24:14.137Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.883275737Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15812023-09-22T23:24:14.140Z{"msg":"[0] client is_active_req TRUE, promote! session 9b5fb6af-ef0d-4296-aeda-f1dea38a8f72","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.883343295Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15822023-09-22T23:24:14.142Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.883720935Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15832023-09-22T23:24:14.145Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.883794729Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15842023-09-22T23:24:14.148Z{"msg":"1762743a-abac-40c6-9217-879068b302ca WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.883848291Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15852023-09-22T23:24:14.151Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.883922119Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15862023-09-22T23:24:14.154Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.884002423Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15872023-09-22T23:24:14.157Z{"msg":"1762743a-abac-40c6-9217-879068b302ca WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.884085122Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15882023-09-22T23:24:14.160Z{"msg":"[2] 1762743a-abac-40c6-9217-879068b302ca (9b5fb6af-ef0d-4296-aeda-f1dea38a8f72) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.886150863Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15892023-09-22T23:24:14.163Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.88623482Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15902023-09-22T23:24:14.165Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:12.886296711Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15912023-09-22T23:24:14.169Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.886346861Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15922023-09-22T23:24:14.171Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.886390707Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15932023-09-22T23:24:14.174Z{"msg":"1762743a-abac-40c6-9217-879068b302ca WaitActive New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.886430823Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15942023-09-22T23:24:14.177Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.886469566Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15952023-09-22T23:24:14.180Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.886507876Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15962023-09-22T23:24:14.183Z{"msg":"[0] 1762743a-abac-40c6-9217-879068b302ca (9b5fb6af-ef0d-4296-aeda-f1dea38a8f72) WaitActive New WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.886549392Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15972023-09-22T23:24:14.186Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.886588757Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15982023-09-22T23:24:14.189Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:12.886627061Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
15992023-09-22T23:24:14.192Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.886646663Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16002023-09-22T23:24:14.195Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.886683432Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16012023-09-22T23:24:14.197Z{"msg":"1762743a-abac-40c6-9217-879068b302ca WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.886721904Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16022023-09-22T23:24:14.200Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.886759968Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16032023-09-22T23:24:14.203Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:12.886779398Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16042023-09-22T23:24:14.987Z{"msg":"[1] 1762743a-abac-40c6-9217-879068b302ca looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.883694644Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204,"looper":"1"}
16052023-09-22T23:24:14.990Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.88380729Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16062023-09-22T23:24:14.993Z{"msg":"[1] 1762743a-abac-40c6-9217-879068b302ca (9b5fb6af-ef0d-4296-aeda-f1dea38a8f72) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.88477551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16072023-09-22T23:24:14.997Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.884867804Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16082023-09-22T23:24:15.000Z{"msg":"[1] client is_active_req TRUE, promote! session 9b5fb6af-ef0d-4296-aeda-f1dea38a8f72","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.884927007Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16092023-09-22T23:24:15.003Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.885468418Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16102023-09-22T23:24:15.006Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.885529347Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16112023-09-22T23:24:15.009Z{"msg":"1762743a-abac-40c6-9217-879068b302ca WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.885554839Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16122023-09-22T23:24:15.012Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.887759634Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16132023-09-22T23:24:15.015Z{"msg":"[1] 1762743a-abac-40c6-9217-879068b302ca (9b5fb6af-ef0d-4296-aeda-f1dea38a8f72) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.888063823Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16142023-09-22T23:24:15.018Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.888124721Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16152023-09-22T23:24:15.021Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:13.888168563Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16162023-09-22T23:24:15.024Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.888208697Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16172023-09-22T23:24:15.027Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.888247884Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16182023-09-22T23:24:15.031Z{"msg":"1762743a-abac-40c6-9217-879068b302ca WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.888287398Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16192023-09-22T23:24:15.034Z{"msg":"[0]R flush_numbers[0..12]: [7, 16, 16, 16, 16, 14, 13, 7, 13, 16, 16, 16]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.888327081Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16202023-09-22T23:24:15.037Z{"msg":"[0]R generation[0..12]: [5, 11, 11, 11, 11, 9, 8, 5, 8, 11, 11, 11]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.888366218Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16212023-09-22T23:24:15.040Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.888419165Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16222023-09-22T23:24:15.044ZThe guest has finished waiting for activation with:12
16232023-09-22T23:24:15.048Z{"msg":"Wait for a query_work_queue command to finish before sending IO
16242023-09-22T23:24:15.051Z[1]R flush_numbers[0..12]: [7, 16, 16, 16, 16, 14, 13, 7, 13, 16, 16, 16]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.888530437Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16252023-09-22T23:24:15.054Z{"msg":"[1]R generation[0..12]: [5, 11, 11, 11, 11, 9, 8, 5, 8, 11, 11, 11]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.888658448Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16262023-09-22T23:24:15.057Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.888733219Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16272023-09-22T23:24:15.060Z{"msg":"[2]R flush_numbers[0..12]: [7, 16, 16, 16, 16, 14, 13, 7, 13, 16, 16, 16]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.888777925Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16282023-09-22T23:24:15.063Z{"msg":"[2]R generation[0..12]: [5, 11, 11, 11, 11, 9, 8, 5, 8, 11, 11, 11]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.888868333Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16292023-09-22T23:24:15.066Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.888910002Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16302023-09-22T23:24:15.069Z{"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.88896288Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16312023-09-22T23:24:15.072Z{"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.88902637Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16322023-09-22T23:24:15.075Z{"msg":"Next flush: 17","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.88907127Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16332023-09-22T23:24:15.078Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.889091303Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16342023-09-22T23:24:15.081Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.889137461Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16352023-09-22T23:24:15.084Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.889159055Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16362023-09-22T23:24:15.087Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.889211984Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16372023-09-22T23:24:15.090Z{"msg":"1762743a-abac-40c6-9217-879068b302ca is now active with session: 9b5fb6af-ef0d-4296-aeda-f1dea38a8f72","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.889275394Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16382023-09-22T23:24:15.093Z{"msg":"1762743a-abac-40c6-9217-879068b302ca Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.889318038Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16392023-09-22T23:24:15.096Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.88938789Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16402023-09-22T23:24:15.099Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.889451001Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16412023-09-22T23:24:15.102Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.889475473Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16422023-09-22T23:24:15.105Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.889528165Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16432023-09-22T23:24:15.108Z{"msg":"[2] Starts cmd_loop","v":0Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
16442023-09-22T23:24:15.111Z,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.889585131Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16452023-09-22T23:24:15.136Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.889606687Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16462023-09-22T23:24:15.139Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.88964325Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16472023-09-22T23:24:15.142Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.889681337Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16482023-09-22T23:24:15.145Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.889719166Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16492023-09-22T23:24:15.147Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.889757314Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16502023-09-22T23:24:15.151Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:13.889794701Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1204}
16512023-09-22T23:24:15.154ZLoading write count information from file "/tmp/test_repair_verify.data"
16522023-09-22T23:24:15.158ZRun Repair workload
16532023-09-22T23:24:15.160Z01/30 Write block 40 len 40960 data: 3 3 4 5 4 3 3 3 3 3
16542023-09-22T23:24:15.163Z02/30 Flush
16552023-09-22T23:24:15.166Z03/30 Write block 457 len 20480 data: 2 3 3 3 4
16562023-09-22T23:24:15.169Z04/30 Flush
16572023-09-22T23:24:15.172Z05/30 Write block 315 len 40960 data: 3 3 3 2 2 3 3 3 3 3
16582023-09-22T23:24:15.174Z06/30 Read block 55 len 12288
16592023-09-22T23:24:15.177Z07/30 Read block 179 len 12288
16602023-09-22T23:24:15.180Z08/30 Read block 308 len 8192
16612023-09-22T23:24:15.183Z09/30 Write block 564 len 24576 data: 2 2 2 2 2 2
16622023-09-22T23:24:15.186Z10/30 Write block 186 len 24576 data: 2 2 2 3 3 2
16632023-09-22T23:24:15.189Z11/30 Read block 486 len 16384
16642023-09-22T23:24:15.191Z12/30 Write block 158 len 16384 data: 3 3 3 3
16652023-09-22T23:24:15.194Z13/30 Read block 585 len 8192
16662023-09-22T23:24:15.197Z14/30 Read block 408 len 36864
16672023-09-22T23:24:15.200Z15/30 Write block 24 len 16384 data: 2 2 2 2
16682023-09-22T23:24:15.202Z16/30 Write block 276 len 16384 data: 3 2 2 3
16692023-09-22T23:24:15.205Z17/30 Read block 541 len 32768
16702023-09-22T23:24:15.208Z18/30 Write block 567 len 40960 data: 3 3 3 2 2 2 2 2 2 2
16712023-09-22T23:24:15.211Z19/30 Read block 94 len 4096
16722023-09-22T23:24:15.214Z20/30 Read block 152 len 36864
16732023-09-22T23:24:15.217Z21/30 Read block 522 len 4096
16742023-09-22T23:24:15.220Z22/30 Read block 168 len 36864
16752023-09-22T23:24:15.223Z23/30 Write block 269 len 32768 data: 4 4 4 4 4 3 3 4
16762023-09-22T23:24:15.226Z24/30 Write block 140 len 20480 data: 2 2 3 3 3
16772023-09-22T23:24:15.229Z25/30 Read block 421 len 32768
16782023-09-22T23:24:15.232Z26/30 Write block 411 len 24576 data: 4 3 3 3 2 2
16792023-09-22T23:24:15.235Z27/30 Read block 138 len 4096
16802023-09-22T23:24:15.238Z28/30 Read block 472 len 32768
16812023-09-22T23:24:15.241Z29/30 Read block 443 len 4096
16822023-09-22T23:24:15.244Z30/30 Write block 344 len 20480 data: 3 3 3 3 2
16832023-09-22T23:24:15.246Z----------------------------------------------------------------
16842023-09-22T23:24:15.249Z Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:28
16852023-09-22T23:24:15.253ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
16862023-09-22T23:24:15.256Z 3 Acked 1002 Write 5 Done Sent Done false
16872023-09-22T23:24:15.260Z 4 Acked 1003 Flush 0 Done Sent Done false
16882023-09-22T23:24:15.263Z 5 Acked 1004 Write 10 Done Sent Done false
16892023-09-22T23:24:15.265Z 6 Acked 1005 Read 3 Done Sent Done false
16902023-09-22T23:24:15.268Z 7 Acked 1006 Read 3 Done Sent Done false
16912023-09-22T23:24:15.271Z 8 Acked 1007 Read 2 Done Sent Done false
16922023-09-22T23:24:15.274Z 9 Acked 1008 Write 6 Done Sent Done false
16932023-09-22T23:24:15.276Z 10 Acked 1009 Write 6 Done Sent Done false
16942023-09-22T23:24:15.279Z 11 Acked 1010 Read 4 Done Sent Done false
16952023-09-22T23:24:15.282Z 12 Acked 1011 Write 4 Done Sent Done false
16962023-09-22T23:24:15.285Z 13 Acked 1012 Read 2 Done Sent Done false
16972023-09-22T23:24:15.287Z 14 Acked 1013 Read 9 Done Sent Done false
16982023-09-22T23:24:15.290Z 15 Acked 1014 Write 4 Done Sent Done false
16992023-09-22T23:24:15.293Z 16 Acked 1015 Write 4 Done Sent Done false
17002023-09-22T23:24:15.295Z 17 Acked 1016 Read 8 Done Sent Done false
17012023-09-22T23:24:15.298Z 18 Acked 1017 Write 10 Done Sent Done false
17022023-09-22T23:24:15.301Z 19 Acked 1018 Read 1 Done Sent Done false
17032023-09-22T23:24:15.304Z 20 Acked 1019 Read 9 Done Sent Done false
17042023-09-22T23:24:15.306Z 21 Acked 1020 Read 1 Done Sent Done false
17052023-09-22T23:24:15.309Z 22 Acked 1021 Read 9 Done Sent Done false
17062023-09-22T23:24:15.312Z 23 Acked 1022 Write 8 Done Sent Done false
17072023-09-22T23:24:15.314Z 24 Acked 1023 Write 5 Done Sent Done false
17082023-09-22T23:24:15.317Z 25 Acked 1024 Read 8 Done Sent Done false
17092023-09-22T23:24:15.320Z 26 Acked 1025 Write 6 Done Sent Done false
17102023-09-22T23:24:15.323Z 27 Acked 1026 Read 1 Done Sent Done false
17112023-09-22T23:24:16.329Z 28 Acked 1027 Read 8 Done Sent Done false
17122023-09-22T23:24:16.333Z 29 Acked 1028 Read 1 Done Sent Done false
17132023-09-22T23:24:16.340Z 30 Acked 1029 Write 5 Sent Sent Sent false
17142023-09-22T23:24:16.342Z STATES DS:0 DS:1 DS:2 TOTAL
17152023-09-22T23:24:16.345Z New 0 0 0 0
17162023-09-22T23:24:16.348Z Sent 1 28 1 30
17172023-09-22T23:24:16.351Z Done 27 0 27 54
17182023-09-22T23:24:16.354Z Skipped 0 0 0 0
17192023-09-22T23:24:16.357Z Error 0 0 0 0
17202023-09-22T23:24:16.360ZLast Flush: 1003 1001 1003
17212023-09-22T23:24:16.363ZDownstairs last five completed: 1001 1000
17222023-09-22T23:24:16.366ZUpstairs last five completed: 30 29 28 27 26 25
17232023-09-22T23:24:16.369ZWrote out file "/tmp/test_repair_verify.data"
17242023-09-22T23:24:16.372Z
17252023-09-22T23:24:16.375ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
17262023-09-22T23:24:16.378Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:14.029315934Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1205}
17272023-09-22T23:24:16.380Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:14.029658891Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1205}
17282023-09-22T23:24:16.383Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:14.029682386Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1205}
17292023-09-22T23:24:16.386Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:14.064937732Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1205}
17302023-09-22T23:24:16.388Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:14.064984062Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1205}
17312023-09-22T23:24:16.395Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:14.065011471Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1205}
17322023-09-22T23:24:16.398Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:14.082408038Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1205}
17332023-09-22T23:24:16.400Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:14.082439793Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1205}
17342023-09-22T23:24:16.406Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:14.082458482Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1205}
17352023-09-22T23:24:16.409ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
17362023-09-22T23:24:16.411Z 0 000-019 5 5 5 7 7 7 F F F
17372023-09-22T23:24:16.414ZError: Difference in extent metadata found!
17382023-09-22T23:24:16.417Z 1 020-039 11 11 11 16 16 16 T F T <---
17392023-09-22T23:24:16.420Z 2 040-059 12 12 12 17 17 17 F F F
17402023-09-22T23:24:16.423Z 3 060-079 11 11 11 16 16 16 F F F
17412023-09-22T23:24:16.426Z 4 080-099 11 11 11 16 16 16 F F F
17422023-09-22T23:24:16.429Z 5 100-119 9 9 9 14 14 14 F F F
17432023-09-22T23:24:16.431Z 6 120-139 8 8 8 13 13 13 F F F
17442023-09-22T23:24:16.434Z 7 140-159 5 5 5 7 7 7 T F T <---
17452023-09-22T23:24:16.437Z 8 160-179 8 8 8 13 13 13 T F T <---
17462023-09-22T23:24:16.440Z 9 180-199 11 11 11 16 16 16 T F T <---
17472023-09-22T23:24:16.443Z 10 200-219 11 11 11 16 16 16 F F F
17482023-09-22T23:24:16.445Z 11 220-239 11 11 11 16 16 16 F F F
17492023-09-22T23:24:16.448Z 12 240-259 7 7 7 11 11 11 F F F
17502023-09-22T23:24:16.451Z 13 260-279 11 11 11 16 16 16 T F T <---
17512023-09-22T23:24:16.454Z 14 280-299 10 10 10 15 15 15 F F F
17522023-09-22T23:24:16.457Z 15 300-319 8 8 8 13 13 13 T F T <---
17532023-09-22T23:24:16.459Z 16 320-339 8 8 8 13 13 13 T F T <---
17542023-09-22T23:24:16.463Z 17 340-359 11 11 11 16 16 16 T F T <---
17552023-09-22T23:24:16.466Z 18 360-379 11 11 11 16 16 16 F F F
17562023-09-22T23:24:16.469Z 19 380-399 10 10 10 15 15 15 F F F
17572023-09-22T23:24:16.471Z 20 400-419 6 6 6 8 8 8 T F T <---
17582023-09-22T23:24:16.474Z 21 420-439 9 9 9 14 14 14 F F F
17592023-09-22T23:24:16.477Z 22 440-459 12 8 12 18 13 18 F F F <---
17602023-09-22T23:24:16.480Z 23 460-479 12 8 12 18 12 18 F F F <---
17612023-09-22T23:24:16.482Z 24 480-499 11 11 11 16 16 16 F F F
17622023-09-22T23:24:16.485Z 25 500-519 11 11 11 16 16 16 F F F
17632023-09-22T23:24:16.488Z 26 520-539 7 7 7 11 11 11 F F F
17642023-09-22T23:24:16.491Z 27 540-559 7 7 7 11 11 11 F F F
17652023-09-22T23:24:16.493Z 28 560-579 1 1 1 1 1 1 T F T <---
17662023-09-22T23:24:16.496Z 29 580-599 9 9 9 14 14 14 F F F
17672023-09-22T23:24:16.504ZMax gen: 12, Max flush: 18
17682023-09-22T23:24:16.506ZA Difference in extent metadata is expected here
17692023-09-22T23:24:16.509ZOn loop 6
17702023-09-22T23:24:16.512Z
17712023-09-22T23:24:16.516ZVerifying data now
17722023-09-22T23:24:21.586ZNo change after verify
17732023-09-22T23:24:21.589ZLoop: 6 Downstairs dump after verify (and repair):
17742023-09-22T23:24:21.604Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.500846551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1210}
17752023-09-22T23:24:21.607Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.501204133Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1210}
17762023-09-22T23:24:21.610Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.501228858Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1210}
17772023-09-22T23:24:21.622Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.519572023Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1210}
17782023-09-22T23:24:21.625Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.519624795Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1210}
17792023-09-22T23:24:21.627Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.519645251Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1210}
17802023-09-22T23:24:21.633Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.530502367Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1210}
17812023-09-22T23:24:21.636Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.530553789Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1210}
17822023-09-22T23:24:21.639Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.530574629Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1210}
17832023-09-22T23:24:21.644ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
17842023-09-22T23:24:21.646Z 0 000-019 5 5 5 7 7 7 F F F
17852023-09-22T23:24:21.649Z 1 020-039 13 13 13 19 19 19 F F F
17862023-09-22T23:24:21.652Z 2 040-059 12 12 12 17 17 17 F F F
17872023-09-22T23:24:21.655Z 3 060-079 11 11 11 16 16 16 F F F
17882023-09-22T23:24:21.657Z 4 080-099 11 11 11 16 16 16 F F F
17892023-09-22T23:24:21.660Z 5 100-119 9 9 9 14 14 14 F F F
17902023-09-22T23:24:21.663Z 6 120-139 8 8 8 13 13 13 F F F
17912023-09-22T23:24:21.666Z 7 140-159 13 13 13 19 19 19 F F F
17922023-09-22T23:24:21.669Z 8 160-179 13 13 13 19 19 19 F F F
17932023-09-22T23:24:21.671Z 9 180-199 13 13 13 19 19 19 F F F
17942023-09-22T23:24:21.674Z 10 200-219 11 11 11 16 16 16 F F F
17952023-09-22T23:24:21.677Z 11 220-239 11 11 11 16 16 16 F F F
17962023-09-22T23:24:21.680Z 12 240-259 7 7 7 11 11 11 F F F
17972023-09-22T23:24:21.683Z 13 260-279 13 13 13 19 19 19 F F F
17982023-09-22T23:24:21.686Z 14 280-299 10 10 10 15 15 15 F F F
17992023-09-22T23:24:21.689Z 15 300-319 13 13 13 19 19 19 F F F
18002023-09-22T23:24:21.691Z 16 320-339 13 13 13 19 19 19 F F F
18012023-09-22T23:24:21.694Z 17 340-359 13 13 13 19 19 19 F F F
18022023-09-22T23:24:21.697Z 18 360-379 11 11 11 16 16 16 F F F
18032023-09-22T23:24:21.700Z 19 380-399 10 10 10 15 15 15 F F F
18042023-09-22T23:24:21.703Z 20 400-419 13 13 13 19 19 19 F F F
18052023-09-22T23:24:21.705Z 21 420-439 9 9 9 14 14 14 F F F
18062023-09-22T23:24:21.708Z 22 440-459 12 12 12 18 18 18 F F F
18072023-09-22T23:24:21.711Z 23 460-479 12 12 12 18 18 18 F F F
18082023-09-22T23:24:21.714Z 24 480-499 11 11 11 16 16 16 F F F
18092023-09-22T23:24:21.717Z 25 500-519 11 11 11 16 16 16 F F F
18102023-09-22T23:24:21.719Z 26 520-539 7 7 7 11 11 11 F F F
18112023-09-22T23:24:21.722Z 27 540-559 7 7 7 11 11 11 F F F
18122023-09-22T23:24:21.725Z 28 560-579 13 13 13 19 19 19 F F F
18132023-09-22T23:24:21.727Z 29 580-599 9 9 9 14 14 14 F F F
18142023-09-22T23:24:21.730ZMax gen: 13, Max flush: 19
18152023-09-22T23:24:21.733Z
18162023-09-22T23:24:21.736ZBegin loop 7
18172023-09-22T23:24:21.738ZDownstairs to restart: 0
18182023-09-22T23:24:21.744Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.578640036Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18192023-09-22T23:24:21.747Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.579030941Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18202023-09-22T23:24:21.749Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.579107593Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18212023-09-22T23:24:21.752Z{"msg":"Crucible stats registered with UUID: fe54da82-44c1-41f7-8a83-54df78c3377d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.579144402Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18222023-09-22T23:24:21.755Z{"msg":"Crucible fe54da82-44c1-41f7-8a83-54df78c3377d has session id: 38e50267-72da-481a-81c9-aa2cb1416ebc","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.579203899Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18232023-09-22T23:24:21.758ZCrucible runtime is spawned
18242023-09-22T23:24:21.761ZThe guest has requested activation with gen:14
18252023-09-22T23:24:21.763Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.579469908Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212,"task":"up_listen"}
18262023-09-22T23:24:21.767Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.579526612Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18272023-09-22T23:24:21.770Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.579566732Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18282023-09-22T23:24:21.773Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.579608812Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212,"looper":"1"}
18292023-09-22T23:24:21.776Z{"msg":"[1] fe54da82-44c1-41f7-8a83-54df78c3377d looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.579877932Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212,"looper":"1"}
18302023-09-22T23:24:21.778Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.579974589Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212,"looper":"2"}
18312023-09-22T23:24:21.782Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.580233596Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212,"looper":"0"}
18322023-09-22T23:24:21.785Z{"msg":"[2] fe54da82-44c1-41f7-8a83-54df78c3377d looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.580295871Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212,"looper":"2"}
18332023-09-22T23:24:21.787Z{"msg":"fe54da82-44c1-41f7-8a83-54df78c3377d active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.580817735Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18342023-09-22T23:24:21.790Z{"msg":"Set desired generation to :14","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.580875771Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18352023-09-22T23:24:21.793Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.580962998Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18362023-09-22T23:24:21.796Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.581356005Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18372023-09-22T23:24:21.799Z{"msg":"[2] fe54da82-44c1-41f7-8a83-54df78c3377d (bb978259-6dbf-4e43-b0a3-5f940e1bd315) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.581719486Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18382023-09-22T23:24:21.802Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.581800733Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18392023-09-22T23:24:21.805Z{"msg":"[2] client is_active_req TRUE, promote! session bb978259-6dbf-4e43-b0a3-5f940e1bd315","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.581878327Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18402023-09-22T23:24:21.807Z{"msg":"[1] fe54da82-44c1-41f7-8a83-54df78c3377d (bb978259-6dbf-4e43-b0a3-5f940e1bd315) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.58195783Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18412023-09-22T23:24:21.810Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.582032233Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18422023-09-22T23:24:21.813Z{"msg":"[1] client is_active_req TRUE, promote! session bb978259-6dbf-4e43-b0a3-5f940e1bd315","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.582111793Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18432023-09-22T23:24:21.816Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.58235882Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18442023-09-22T23:24:21.819Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.582442843Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18452023-09-22T23:24:21.822Z{"msg":"fe54da82-44c1-41f7-8a83-54df78c3377d New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.582512872Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18462023-09-22T23:24:21.825Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.582591818Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18472023-09-22T23:24:21.828Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.582668976Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18482023-09-22T23:24:21.830Z{"msg":"fe54da82-44c1-41f7-8a83-54df78c3377d New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.582743241Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18492023-09-22T23:24:21.834Z{"msg":"[2] fe54da82-44c1-41f7-8a83-54df78c3377d (bb978259-6dbf-4e43-b0a3-5f940e1bd315) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.584683563Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18502023-09-22T23:24:21.837Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.58474453Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18512023-09-22T23:24:21.840Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:20.584873717Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18522023-09-22T23:24:21.843Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.584895249Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18532023-09-22T23:24:21.846Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.584913545Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18542023-09-22T23:24:21.849Z{"msg":"fe54da82-44c1-41f7-8a83-54df78c3377d New WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.58495106Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18552023-09-22T23:24:21.852Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.584990825Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18562023-09-22T23:24:21.855Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.585030388Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18572023-09-22T23:24:21.857Z{"msg":"[1] fe54da82-44c1-41f7-8a83-54df78c3377d (bb978259-6dbf-4e43-b0a3-5f940e1bd315) New WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.585072688Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18582023-09-22T23:24:21.860Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.585112478Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18592023-09-22T23:24:21.863Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:20.585132289Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18602023-09-22T23:24:21.867Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.585169483Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18612023-09-22T23:24:21.869Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.585207945Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18622023-09-22T23:24:21.872Z{"msg":"fe54da82-44c1-41f7-8a83-54df78c3377d New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.585246583Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18632023-09-22T23:24:21.875Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.585268295Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18642023-09-22T23:24:21.878Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:20.585303806Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18652023-09-22T23:24:22.685Z{"msg":"[0] fe54da82-44c1-41f7-8a83-54df78c3377d looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.582095265Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212,"looper":"0"}
18662023-09-22T23:24:22.688Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.582186314Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18672023-09-22T23:24:22.691Z{"msg":"[0] fe54da82-44c1-41f7-8a83-54df78c3377d (bb978259-6dbf-4e43-b0a3-5f940e1bd315) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.583307122Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18682023-09-22T23:24:22.694Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.583365125Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18692023-09-22T23:24:22.697Z{"msg":"[0] client is_active_req TRUE, promote! session bb978259-6dbf-4e43-b0a3-5f940e1bd315","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.583390694Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18702023-09-22T23:24:22.700Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.583961142Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18712023-09-22T23:24:22.703Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.584018398Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18722023-09-22T23:24:22.706Z{"msg":"fe54da82-44c1-41f7-8a83-54df78c3377d WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.584042151Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18732023-09-22T23:24:22.709Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.5862216Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18742023-09-22T23:24:22.711Z{"msg":"[0] fe54da82-44c1-41f7-8a83-54df78c3377d (bb978259-6dbf-4e43-b0a3-5f940e1bd315) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.586399284Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18752023-09-22T23:24:22.714Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.586465543Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18762023-09-22T23:24:22.717Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:21.586523295Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18772023-09-22T23:24:22.721Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.586578071Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18782023-09-22T23:24:22.724Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.586632966Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18792023-09-22T23:24:22.727Z{"msg":"fe54da82-44c1-41f7-8a83-54df78c3377d WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.586688893Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18802023-09-22T23:24:22.730Z{"msg":"[0]R flush_numbers[0..12]: [7, 19, 17, 16, 16, 14, 13, 19, 19, 19, 16, 16]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.586744359Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18812023-09-22T23:24:22.732Z{"msg":"[0]R generation[0..12]: [5, 13, 12, 11, 11, 9, 8, 13, 13, 13, 11, 11]","v":0,"name":"crucible","level"The guest has finished waiting for activation with:14
18822023-09-22T23:24:22.735ZWait for a query_work_queue command to finish before sending IO
18832023-09-22T23:24:22.738Z:30,"time":"2023-09-22T23:24:21.58693655Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18842023-09-22T23:24:22.741Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.586987352Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18852023-09-22T23:24:22.743Z{"msg":"[1]R flush_numbers[0..12]: [7, 19, 17, 16, 16, 14, 13, 19, 19, 19, 16, 16]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.587073021Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18862023-09-22T23:24:22.746Z{"msg":"[1]R generation[0..12]: [5, 13, 12, 11, 11, 9, 8, 13, 13, 13, 11, 11]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.587147181Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18872023-09-22T23:24:22.750Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.587200489Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18882023-09-22T23:24:22.753Z{"msg":"[2]R flush_numbers[0..12]: [7, 19, 17, 16, 16, 14, 13, 19, 19, 19, 16, 16]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.587262663Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18892023-09-22T23:24:22.758Z{"msg":"[2]R generation[0..12]: [5, 13, 12, 11, 11, 9, 8, 13, 13, 13, 11, 11]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.587327039Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18902023-09-22T23:24:22.762Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.587382429Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18912023-09-22T23:24:22.765Z{"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.587444809Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18922023-09-22T23:24:22.768Z{"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.587503676Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18932023-09-22T23:24:22.771Z{"msg":"Next flush: 20","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.587571344Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18942023-09-22T23:24:22.773Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.587636122Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18952023-09-22T23:24:22.776Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.587688049Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18962023-09-22T23:24:22.779Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.587743284Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18972023-09-22T23:24:22.782Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.58780161Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
18982023-09-22T23:24:22.785Z{"msg":"fe54da82-44c1-41f7-8a83-54df78c3377d is now active with session: bb978259-6dbf-4e43-b0a3-5f940e1bd315","v":0,"name":"crucible"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
18992023-09-22T23:24:22.788Z,"level":30,"time":"2023-09-22T23:24:21.587896344Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
19002023-09-22T23:24:22.790Z{"msg":"fe54da82-44c1-41f7-8a83-54df78c3377d Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.587954988Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
19012023-09-22T23:24:22.793Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.588034082Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
19022023-09-22T23:24:22.796Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.588091971Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
19032023-09-22T23:24:22.799Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.588145423Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
19042023-09-22T23:24:22.802Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.588198282Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
19052023-09-22T23:24:22.805Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.588251112Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
19062023-09-22T23:24:22.808Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.588303659Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
19072023-09-22T23:24:22.811Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.588355556Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
19082023-09-22T23:24:22.814Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30Loading write count information from file "/tmp/test_repair_verify.data"
19092023-09-22T23:24:22.817Z,"time":"2023-09-22T23:24:21.588407837Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":Run Repair workload
19102023-09-22T23:24:22.820Z1212}
19112023-09-22T23:24:22.822Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.588565785Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
19122023-09-22T23:24:22.826Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.58862123Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
19132023-09-22T23:24:22.829Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.588674118Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1212}
19142023-09-22T23:24:22.832Z01/30 Write block 60 len 8192 data: 2 2
19152023-09-22T23:24:22.835Z02/30 Write block 89 len 32768 data: 4 4 3 3 3 3 4 4
19162023-09-22T23:24:22.837Z03/30 Write block 63 len 40960 data: 3 3 3 3 5 6 6 5 5 5
19172023-09-22T23:24:22.840Z04/30 Write block 427 len 28672 data: 3 3 3 3 3 3 2
19182023-09-22T23:24:22.843Z05/30 Write block 512 len 12288 data: 3 3 3
19192023-09-22T23:24:22.846Z06/30 Read block 486 len 4096
19202023-09-22T23:24:22.849Z07/30 Read block 514 len 28672
19212023-09-22T23:24:22.852Z08/30 Read block 237 len 4096
19222023-09-22T23:24:22.855Z09/30 Flush
19232023-09-22T23:24:22.858Z10/30 Write block 60 len 40960 data: 3 3 3 4 4 4 4 6 7 7
19242023-09-22T23:24:22.861Z11/30 Write block 527 len 36864 data: 2 2 2 2 3 3 3 3 3
19252023-09-22T23:24:22.864Z12/30 Write block 400 len 40960 data: 2 2 3 3 3 3 3 3 3 4
19262023-09-22T23:24:22.867Z13/30 Read block 107 len 28672
19272023-09-22T23:24:22.869Z14/30 Flush
19282023-09-22T23:24:22.873Z15/30 Write block 556 len 12288 data: 2 2 2
19292023-09-22T23:24:22.876Z16/30 Flush
19302023-09-22T23:24:22.878Z17/30 Flush
19312023-09-22T23:24:22.881Z18/30 Write block 360 len 32768 data: 3 2 2 2 2 2 3 3
19322023-09-22T23:24:22.884Z19/30 Write block 211 len 40960 data: 4 4 4 3 3 2 2 2 2 2
19332023-09-22T23:24:22.887Z20/30 Read block 146 len 28672
19342023-09-22T23:24:22.890Z21/30 Read block 359 len 20480
19352023-09-22T23:24:22.892Z22/30 Write block 313 len 36864 data: 2 3 4 4 4 3 3 4 4
19362023-09-22T23:24:22.896Z23/30 Write block 290 len 28672 data: 4 4 3 3 3 3 3
19372023-09-22T23:24:22.898Z24/30 Write block 220 len 8192 data: 3 2
19382023-09-22T23:24:22.903Z25/30 Write block 546 len 4096 data: 2
19392023-09-22T23:24:22.906Z26/30 Write block 120 len 20480 data: 2 2 3 3 3
19402023-09-22T23:24:22.909Z27/30 Write block 552 len 40960 data: 3 3 2 2 3 3 3 2 2 2
19412023-09-22T23:24:22.912Z28/30 Write block 549 len 20480 data: 3 3 3 4 4
19422023-09-22T23:24:22.915Z29/30 Write block 22 len 40960 data: 3 2 3 3 3 3 2 2 2 2
19432023-09-22T23:24:22.918Z30/30 Read block 508 len 24576
19442023-09-22T23:24:22.922Z----------------------------------------------------------------
19452023-09-22T23:24:22.924Z Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:30
19462023-09-22T23:24:22.927ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
19472023-09-22T23:24:22.930Z 1 Acked 1000 Write 2 Done Done Done false
19482023-09-22T23:24:22.933Z 2 Acked 1001 Write 8 Done Done Done false
19492023-09-22T23:24:22.936Z 3 Acked 1002 Write 10 Done Done Done false
19502023-09-22T23:24:22.939Z 4 Acked 1003 Write 7 Done Done Done false
19512023-09-22T23:24:22.942Z 5 Acked 1004 Write 3 Done Done Done false
19522023-09-22T23:24:22.945Z 6 Acked 1005 Read 1 Done Done Done false
19532023-09-22T23:24:22.948Z 7 Acked 1006 Read 7 Sent Done Done false
19542023-09-22T23:24:22.950Z 8 Acked 1007 Read 1 Sent Done Done false
19552023-09-22T23:24:22.953Z 9 Acked 1008 Flush 0 Sent Done Done false
19562023-09-22T23:24:22.956Z 10 Acked 1009 Write 10 Sent Done Done false
19572023-09-22T23:24:22.959Z 11 Acked 1010 Write 9 Sent Done Done false
19582023-09-22T23:24:22.962Z 12 Acked 1011 Write 10 Sent Done Done false
19592023-09-22T23:24:22.965Z 13 Acked 1012 Read 7 Sent Done Done false
19602023-09-22T23:24:22.968Z 14 Acked 1013 Flush 0 Sent Done Done false
19612023-09-22T23:24:22.971Z 15 Acked 1014 Write 3 Sent Done Done false
19622023-09-22T23:24:22.973Z 16 Acked 1015 Flush 0 Sent Done Done false
19632023-09-22T23:24:22.976Z 17 Acked 1016 Flush 0 Sent Done Done false
19642023-09-22T23:24:22.979Z 18 Acked 1017 Write 8 Sent Done Done false
19652023-09-22T23:24:22.982Z 19 Acked 1018 Write 10 Sent Done Done false
19662023-09-22T23:24:22.985Z 20 Acked 1019 Read 7 Sent Done Done false
19672023-09-22T23:24:22.989Z 21 Acked 1020 Read 5 Sent Done Done false
19682023-09-22T23:24:22.994Z 22 Acked 1021 Write 9 Sent Done Done false
19692023-09-22T23:24:22.999Z 23 Acked 1022 Write 7 Sent Done Done false
19702023-09-22T23:24:23.005Z 24 Acked 1023 Write 2 Sent Done Done false
19712023-09-22T23:24:23.008Z 25 Acked 1024 Write 1 Sent Done Done false
19722023-09-22T23:24:23.011Z 26 Acked 1025 Write 5 Sent Done Done false
19732023-09-22T23:24:23.014Z 27 Acked 1026 Write 10 Sent Done Done false
19742023-09-22T23:24:23.017Z 28 Acked 1027 Write 5 Sent Done Done false
19752023-09-22T23:24:23.020Z 29 Acked 1028 Write 10 Sent Done Done false
19762023-09-22T23:24:23.023Z 30 Acked 1029 Read 6 Sent Done Done false
19772023-09-22T23:24:23.025Z STATES DS:0 DS:1 DS:2 TOTAL
19782023-09-22T23:24:23.028Z New 0 0 0 0
19792023-09-22T23:24:23.031Z Sent 24 0 0 24
19802023-09-22T23:24:23.034Z Done 6 30 30 66
19812023-09-22T23:24:23.036Z Skipped 0 0 0 0
19822023-09-22T23:24:23.039Z Error 0 0 0 0
19832023-09-22T23:24:23.042ZLast Flush: 0 1016 1016
19842023-09-22T23:24:23.045ZDownstairs last five completed:
19852023-09-22T23:24:23.047ZUpstairs last five completed: 30 29 28 27 26 25
19862023-09-22T23:24:23.050ZWrote out file "/tmp/test_repair_verify.data"
19872023-09-22T23:24:23.053Z
19882023-09-22T23:24:23.056ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
19892023-09-22T23:24:23.058Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.73113283Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1213}
19902023-09-22T23:24:23.062Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.73150245Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1213}
19912023-09-22T23:24:23.065Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.731525982Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1213}
19922023-09-22T23:24:23.068Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.773858822Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1213}
19932023-09-22T23:24:23.071Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.773915061Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1213}
19942023-09-22T23:24:23.074Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.77395393Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1213}
19952023-09-22T23:24:23.077Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.806781599Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1213}
19962023-09-22T23:24:23.080Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.806857921Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1213}
19972023-09-22T23:24:23.083Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:21.806916984Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1213}
19982023-09-22T23:24:23.086ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
19992023-09-22T23:24:23.088Z 0 000-019 5 5 5 7 7 7 F F F
20002023-09-22T23:24:23.091Z 1 020-039 13 13 13 19 19 19 F T T <---
20012023-09-22T23:24:23.094Z 2 040-059 12 12 12 17 17 17 F F F
20022023-09-22T23:24:23.097Z 3 060-079 11 14 14 16 21 21 T F F <---
20032023-09-22T23:24:23.100Z 4 080-099 11 14 14 16 20 20 T F F <---
20042023-09-22T23:24:23.103Z 5 100-119 9 9 9 14 14 14 F F F
20052023-09-22T23:24:23.105Z 6 120-139 8 8 8 13 13 13 F T T <---
20062023-09-22T23:24:23.108Z 7 140-159 13 13 13 19 19 19 F F F
20072023-09-22T23:24:23.112Z 8 160-179 13 13 13 19 19 19 F F F
20082023-09-22T23:24:23.114Z 9 180-199 13 13 13 19 19 19 F F F
20092023-09-22T23:24:23.117Z 10 200-219 11 11 11 16 16 16 F T T <---
20102023-09-22T23:24:23.120Z 11 220-239 11 11 11 16 16 16 F T T <---
20112023-09-22T23:24:23.122Z 12 240-259 7 7 7 11 11 11 F F F
20122023-09-22T23:24:23.125Z 13 260-279 13 13 13 19 19 19 F F F
20132023-09-22T23:24:23.128Z 14 280-299 10 10 10 15 15 15 F T T <---
20142023-09-22T23:24:23.131ZError: Difference in extent metadata found!
20152023-09-22T23:24:23.134Z 15 300-319 13 13 13 19 19 19 F T T <---
20162023-09-22T23:24:23.137Z 16 320-339 13 13 13 19 19 19 F T T <---
20172023-09-22T23:24:23.140Z 17 340-359 13 13 13 19 19 19 F F F
20182023-09-22T23:24:23.142Z 18 360-379 11 11 11 16 16 16 F T T <---
20192023-09-22T23:24:23.146Z 19 380-399 10 10 10 15 15 15 F F F
20202023-09-22T23:24:23.148Z 20 400-419 13 14 14 19 21 21 F F F <---
20212023-09-22T23:24:23.151Z 21 420-439 9 14 14 14 20 20 T F F <---
20222023-09-22T23:24:23.154Z 22 440-459 12 12 12 18 18 18 F F F
20232023-09-22T23:24:23.157Z 23 460-479 12 12 12 18 18 18 F F F
20242023-09-22T23:24:23.159Z 24 480-499 11 11 11 16 16 16 F F F
20252023-09-22T23:24:23.162Z 25 500-519 11 14 14 16 20 20 T F F <---
20262023-09-22T23:24:23.165Z 26 520-539 7 14 14 11 21 21 F F F <---
20272023-09-22T23:24:23.168Z 27 540-559 7 14 14 11 22 22 F T T <---
20282023-09-22T23:24:23.171Z 28 560-579 13 13 13 19 19 19 F T T <---
20292023-09-22T23:24:23.174Z 29 580-599 9 9 9 14 14 14 F F F
20302023-09-22T23:24:23.177ZMax gen: 14, Max flush: 22
20312023-09-22T23:24:23.179ZA Difference in extent metadata is expected here
20322023-09-22T23:24:23.183ZOn loop 7
20332023-09-22T23:24:23.185Z
20342023-09-22T23:24:23.188ZVerifying data now
20352023-09-22T23:24:29.620ZNo change after verify
20362023-09-22T23:24:29.623ZLoop: 7 Downstairs dump after verify (and repair):
20372023-09-22T23:24:29.638Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.534669339Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1218}
20382023-09-22T23:24:29.641Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.535072855Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1218}
20392023-09-22T23:24:29.644Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.535098265Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1218}
20402023-09-22T23:24:29.656Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.553558823Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1218}
20412023-09-22T23:24:29.659Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.553610952Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1218}
20422023-09-22T23:24:29.662Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.553631664Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1218}
20432023-09-22T23:24:29.667Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.564390061Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1218}
20442023-09-22T23:24:29.670Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.564442374Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1218}
20452023-09-22T23:24:29.673Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.564463094Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1218}
20462023-09-22T23:24:29.678ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
20472023-09-22T23:24:29.681Z 0 000-019 5 5 5 7 7 7 F F F
20482023-09-22T23:24:29.684Z 1 020-039 15 15 15 23 23 23 F F F
20492023-09-22T23:24:29.687Z 2 040-059 12 12 12 17 17 17 F F F
20502023-09-22T23:24:29.691Z 3 060-079 14 14 14 21 21 21 F F F
20512023-09-22T23:24:29.694Z 4 080-099 14 14 14 20 20 20 F F F
20522023-09-22T23:24:29.697Z 5 100-119 9 9 9 14 14 14 F F F
20532023-09-22T23:24:29.700Z 6 120-139 15 15 15 23 23 23 F F F
20542023-09-22T23:24:29.703Z 7 140-159 13 13 13 19 19 19 F F F
20552023-09-22T23:24:29.705Z 8 160-179 13 13 13 19 19 19 F F F
20562023-09-22T23:24:29.708Z 9 180-199 13 13 13 19 19 19 F F F
20572023-09-22T23:24:29.711Z 10 200-219 15 15 15 23 23 23 F F F
20582023-09-22T23:24:29.714Z 11 220-239 15 15 15 23 23 23 F F F
20592023-09-22T23:24:29.717Z 12 240-259 7 7 7 11 11 11 F F F
20602023-09-22T23:24:29.720Z 13 260-279 13 13 13 19 19 19 F F F
20612023-09-22T23:24:29.723Z 14 280-299 15 15 15 23 23 23 F F F
20622023-09-22T23:24:29.726Z 15 300-319 15 15 15 23 23 23 F F F
20632023-09-22T23:24:29.729Z 16 320-339 15 15 15 23 23 23 F F F
20642023-09-22T23:24:29.732Z 17 340-359 13 13 13 19 19 19 F F F
20652023-09-22T23:24:29.735Z 18 360-379 15 15 15 23 23 23 F F F
20662023-09-22T23:24:29.738Z 19 380-399 10 10 10 15 15 15 F F F
20672023-09-22T23:24:29.741Z 20 400-419 14 14 14 21 21 21 F F F
20682023-09-22T23:24:29.744Z 21 420-439 14 14 14 20 20 20 F F F
20692023-09-22T23:24:29.747Z 22 440-459 12 12 12 18 18 18 F F F
20702023-09-22T23:24:29.751Z 23 460-479 12 12 12 18 18 18 F F F
20712023-09-22T23:24:29.754Z 24 480-499 11 11 11 16 16 16 F F F
20722023-09-22T23:24:29.757Z 25 500-519 14 14 14 20 20 20 F F F
20732023-09-22T23:24:29.759Z 26 520-539 14 14 14 21 21 21 F F F
20742023-09-22T23:24:29.762Z 27 540-559 15 15 15 23 23 23 F F F
20752023-09-22T23:24:29.766Z 28 560-579 15 15 15 23 23 23 F F F
20762023-09-22T23:24:29.769Z 29 580-599 9 9 9 14 14 14 F F F
20772023-09-22T23:24:29.772ZMax gen: 15, Max flush: 23
20782023-09-22T23:24:29.774Z
20792023-09-22T23:24:29.777ZBegin loop 8
20802023-09-22T23:24:29.780ZDownstairs to restart: 2
20812023-09-22T23:24:29.783Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.612903711Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
20822023-09-22T23:24:29.786Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.613271786Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
20832023-09-22T23:24:29.788Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.613323799Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
20842023-09-22T23:24:29.791Z{"msg":"Crucible stats registered with UUID: 2e0d1d39-45f7-48ef-8034-a3ccf4839c2c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.613348797Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
20852023-09-22T23:24:29.794Z{"msg":"Crucible 2e0d1d39-45f7-48ef-8034-a3ccf4839c2c has session id: d42df077-1169-4d0c-8c61-4bbcbe5cec91","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.613388999Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
20862023-09-22T23:24:29.797ZCrucible runtime is spawned
20872023-09-22T23:24:29.799ZThe guest has requested activation with gen:16
20882023-09-22T23:24:29.802Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.613718355Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220,"task":"up_listen"}
20892023-09-22T23:24:29.805Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.613797299Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
20902023-09-22T23:24:29.808Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.613856023Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
20912023-09-22T23:24:29.812Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.613895275Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220,"looper":"2"}
20922023-09-22T23:24:29.815Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.614083173Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220,"looper":"0"}
20932023-09-22T23:24:29.818Z{"msg":"[0] 2e0d1d39-45f7-48ef-8034-a3ccf4839c2c looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.614519797Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220,"looper":"0"}
20942023-09-22T23:24:29.821Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.614665349Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220,"looper":"1"}
20952023-09-22T23:24:29.824Z{"msg":"[1] 2e0d1d39-45f7-48ef-8034-a3ccf4839c2c looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.614707754Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220,"looper":"1"}
20962023-09-22T23:24:29.827Z{"msg":"2e0d1d39-45f7-48ef-8034-a3ccf4839c2c active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.615041766Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
20972023-09-22T23:24:29.830Z{"msg":"Set desired generation to :16","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.615093466Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
20982023-09-22T23:24:29.833Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.615261909Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
20992023-09-22T23:24:29.836Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.615690929Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21002023-09-22T23:24:29.839Z{"msg":"[0] 2e0d1d39-45f7-48ef-8034-a3ccf4839c2c (132bb753-b29b-4880-9745-f4894a71c07f) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.615965801Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21012023-09-22T23:24:29.842Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.616060362Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21022023-09-22T23:24:29.845Z{"msg":"[0] client is_active_req TRUE, promote! session 132bb753-b29b-4880-9745-f4894a71c07f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.616134844Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21032023-09-22T23:24:29.848Z{"msg":"[1] 2e0d1d39-45f7-48ef-8034-a3ccf4839c2c (132bb753-b29b-4880-9745-f4894a71c07f) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.616208661Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21042023-09-22T23:24:29.851Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.616277231Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21052023-09-22T23:24:29.854Z{"msg":"[1] client is_active_req TRUE, promote! session 132bb753-b29b-4880-9745-f4894a71c07f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.616333324Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21062023-09-22T23:24:29.857Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.616756909Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21072023-09-22T23:24:29.859Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.61681581Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21082023-09-22T23:24:29.862Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.616844703Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21092023-09-22T23:24:29.865Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.616868448Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21102023-09-22T23:24:29.868Z{"msg":"2e0d1d39-45f7-48ef-8034-a3ccf4839c2c WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.616889577Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21112023-09-22T23:24:29.871Z{"msg":"2e0d1d39-45f7-48ef-8034-a3ccf4839c2c WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.616929791Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21122023-09-22T23:24:29.874Z{"msg":"[0] 2e0d1d39-45f7-48ef-8034-a3ccf4839c2c (132bb753-b29b-4880-9745-f4894a71c07f) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.619122757Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21132023-09-22T23:24:29.877Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.619201453Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21142023-09-22T23:24:29.880Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:28.619271085Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21152023-09-22T23:24:29.883Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.619342218Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21162023-09-22T23:24:29.885Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.619414144Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21172023-09-22T23:24:29.888Z{"msg":"2e0d1d39-45f7-48ef-8034-a3ccf4839c2c WaitQuorum WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.619464849Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21182023-09-22T23:24:29.891Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.619504519Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21192023-09-22T23:24:29.894Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.619542331Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21202023-09-22T23:24:29.897Z{"msg":"[1] 2e0d1d39-45f7-48ef-8034-a3ccf4839c2c (132bb753-b29b-4880-9745-f4894a71c07f) WaitQuorum WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.620297898Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21212023-09-22T23:24:29.899Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.620378749Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21222023-09-22T23:24:29.902Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:28.620438498Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21232023-09-22T23:24:29.905Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.620493378Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21242023-09-22T23:24:29.908Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.620531115Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21252023-09-22T23:24:29.911Z{"msg":"2e0d1d39-45f7-48ef-8034-a3ccf4839c2c WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.620567055Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21262023-09-22T23:24:29.914Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.620625478Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21272023-09-22T23:24:29.917Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:28.620660149Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21282023-09-22T23:24:30.718Z{"msg":"[2] 2e0d1d39-45f7-48ef-8034-a3ccf4839c2c looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.615395731Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220,"looper":"2"}
21292023-09-22T23:24:30.722Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.615481905Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21302023-09-22T23:24:30.725Z{"msg":"[2] 2e0d1d39-45f7-48ef-8034-a3ccf4839c2c (132bb753-b29b-4880-9745-f4894a71c07f) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.616603642Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21312023-09-22T23:24:30.728Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.616664116Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21322023-09-22T23:24:30.731Z{"msg":"[2] client is_active_req TRUE, promote! session 132bb753-b29b-4880-9745-f4894a71c07f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.616709403Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21332023-09-22T23:24:30.734Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.617282552Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21342023-09-22T23:24:30.737Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.617340304Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21352023-09-22T23:24:30.740Z{"msg":"2e0d1d39-45f7-48ef-8034-a3ccf4839c2c WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.617382857Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21362023-09-22T23:24:30.744Z{"msg":"[2] 2e0d1d39-45f7-48ef-8034-a3ccf4839c2c (132bb753-b29b-4880-9745-f4894a71c07f) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.619730599Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21372023-09-22T23:24:30.747Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.619783546Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21382023-09-22T23:24:30.750Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:29.619805574Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21392023-09-22T23:24:30.753Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.619823338Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21402023-09-22T23:24:30.756Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.619841349Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21412023-09-22T23:24:30.759Z{"msg":"2e0d1d39-45f7-48ef-8034-a3ccf4839c2c WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.619860281Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21422023-09-22T23:24:30.762Z{"msg":"[0]R flush_numbers[0..12]: [7, 23, 17, 21, 20, 14, 23, 19, 19, 19, 23, 23]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.619899238Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21432023-09-22T23:24:30.765Z{"msg":"[0]R generation[0..12]: [5, 15, 12, 14, 14, 9, 15, 13, 13, 13, 15, 15]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.619939193Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21442023-09-22T23:24:30.772Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.619960673Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21452023-09-22T23:24:30.775Z{"msg":"[1]R flush_numbers[0..12]: [7, 23, 17, 21, 20, 14, 23, 19, 19, 19, 23, 23]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.620001992Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21462023-09-22T23:24:30.778Z{"msg":"[1]R generation[0..12]: [5, 15, 12, 14, 14, 9, 15, 13, 13, 13, 15, 15]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.620058458Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21472023-09-22T23:24:30.781Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:16
21482023-09-22T23:24:30.785ZWait for a query_work_queue command to finish before sending IO
21492023-09-22T23:24:30.788Z,"time":"2023-09-22T23:24:29.620125896Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21502023-09-22T23:24:30.792Z{"msg":"[2]R flush_numbers[0..12]: [7, 23, 17, 21, 20, 14, 23, 19, 19, 19, 23, 23]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.62032254Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21512023-09-22T23:24:30.795Z{"msg":"[2]R generation[0..12]: [5, 15, 12, 14, 14, 9, 15, 13, 13, 13, 15, 15]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.620372876Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21522023-09-22T23:24:30.798Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.620431989Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21532023-09-22T23:24:30.801Z{"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.620487228Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21542023-09-22T23:24:30.805Z{"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.620549971Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21552023-09-22T23:24:30.808Z{"msg":"Next flush: 24","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.620666289Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21562023-09-22T23:24:30.811Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.620720316Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21572023-09-22T23:24:30.814Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.62078139Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21582023-09-22T23:24:30.817Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.620844785Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21592023-09-22T23:24:30.820Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.620907814Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21602023-09-22T23:24:30.822Z{"msg":"2e0d1d39-45f7-48ef-8034-a3ccf4839c2c is now active with session: 132bb753-b29b-4880-9745-f4894a71c07f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.620963548Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21612023-09-22T23:24:30.825Z{"msg":"2e0d1d39-45f7-48ef-8034-a3ccf4839c2c Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.621027304Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21622023-09-22T23:24:30.828Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.621062692Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21632023-09-22T23:24:30.831Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.621091207Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21642023-09-22T23:24:30.834Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.621110194Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21652023-09-22T23:24:30.837Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.621148915Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21662023-09-22T23:24:30.840Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.621188347Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21672023-09-22T23:24:30.843Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.621227698Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21682023-09-22T23:24:30.846Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
21692023-09-22T23:24:30.849Z,"time":"2023-09-22T23:24:29.621265947Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21702023-09-22T23:24:30.852Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.621374985Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21712023-09-22T23:24:30.854Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.621419989Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21722023-09-22T23:24:30.858Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.621459318Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21732023-09-22T23:24:30.861Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.621497007Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1220}
21742023-09-22T23:24:30.865ZLoading write count information from file "/tmp/test_repair_verify.data"
21752023-09-22T23:24:30.868ZRun Repair workload
21762023-09-22T23:24:30.871Z01/30 Write block 469 len 4096 data: 3
21772023-09-22T23:24:30.874Z02/30 Write block 314 len 32768 data: 4 5 5 5 4 4 5 5
21782023-09-22T23:24:30.877Z03/30 Read block 199 len 16384
21792023-09-22T23:24:30.880Z04/30 Write block 125 len 28672 data: 3 3 4 3 2 2 2
21802023-09-22T23:24:30.882Z05/30 Write block 345 len 16384 data: 4 4 4 3
21812023-09-22T23:24:30.885Z06/30 Read block 401 len 4096
21822023-09-22T23:24:30.888Z07/30 Read block 133 len 40960
21832023-09-22T23:24:30.891Z08/30 Write block 537 len 32768 data: 3 3 3 3 2 2 2 2
21842023-09-22T23:24:30.894Z09/30 Write block 247 len 16384 data: 2 2 2 2
21852023-09-22T23:24:30.896Z10/30 Read block 169 len 36864
21862023-09-22T23:24:30.899Z11/30 Read block 292 len 16384
21872023-09-22T23:24:30.902Z12/30 Write block 105 len 36864 data: 3 3 3 3 3 3 3 3 2
21882023-09-22T23:24:30.905Z13/30 Write block 81 len 8192 data: 6 6
21892023-09-22T23:24:30.908Z14/30 Read block 466 len 4096
21902023-09-22T23:24:30.911Z15/30 Write block 136 len 4096 data: 2
21912023-09-22T23:24:30.914Z16/30 Read block 250 len 36864
21922023-09-22T23:24:30.917Z17/30 Read block 514 len 24576
21932023-09-22T23:24:30.920Z18/30 Read block 296 len 4096
21942023-09-22T23:24:30.922Z19/30 Read block 510 len 12288
21952023-09-22T23:24:30.926Z20/30 Flush
21962023-09-22T23:24:30.929Z21/30 Write block 20 len 36864 data: 2 2 4 3 4 4 4 4 3
21972023-09-22T23:24:30.931Z22/30 Write block 286 len 40960 data: 3 3 4 4 5 5 4 4 4 4
21982023-09-22T23:24:30.934Z23/30 Read block 577 len 24576
21992023-09-22T23:24:30.938Z24/30 Read block 121 len 36864
22002023-09-22T23:24:30.941Z25/30 Read block 583 len 40960
22012023-09-22T23:24:30.943Z26/30 Write block 270 len 28672 data: 5 5 5 5 4 4 5
22022023-09-22T23:24:30.946Z27/30 Write block 394 len 36864 data: 2 2 2 2 2 2 3 3 4
22032023-09-22T23:24:30.949Z28/30 Read block 329 len 16384
22042023-09-22T23:24:30.952Z29/30 Write block 278 len 12288 data: 3 4 3
22052023-09-22T23:24:30.955Z30/30 Read block 179 len 16384
22062023-09-22T23:24:30.958Z----------------------------------------------------------------
22072023-09-22T23:24:30.961Z Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30
22082023-09-22T23:24:30.964ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
22092023-09-22T23:24:30.967Z 1 Acked 1000 Write 1 Done Done Done false
22102023-09-22T23:24:30.969Z 2 Acked 1001 Write 8 Done Done Done false
22112023-09-22T23:24:30.973Z 3 Acked 1002 Read 4 Done Done Done false
22122023-09-22T23:24:30.975Z 4 Acked 1003 Write 7 Done Done Done false
22132023-09-22T23:24:30.978Z 5 Acked 1004 Write 4 Done Done Sent false
22142023-09-22T23:24:30.981Z 6 Acked 1005 Read 1 Done Done Sent false
22152023-09-22T23:24:30.984Z 7 Acked 1006 Read 10 Done Done Sent false
22162023-09-22T23:24:30.987Z 8 Acked 1007 Write 8 Done Done Sent false
22172023-09-22T23:24:30.990Z 9 Acked 1008 Write 4 Done Done Sent false
22182023-09-22T23:24:30.994Z 10 Acked 1009 Read 9 Done Done Sent false
22192023-09-22T23:24:30.996Z 11 Acked 1010 Read 4 Done Done Sent false
22202023-09-22T23:24:30.999Z 12 Acked 1011 Write 9 Done Done Sent false
22212023-09-22T23:24:32.006Z 13 Acked 1012 Write 2 Done Done Sent false
22222023-09-22T23:24:32.009Z 14 Acked 1013 Read 1 Done Done Sent false
22232023-09-22T23:24:32.013Z 15 Acked 1014 Write 1 Done Done Sent false
22242023-09-22T23:24:32.016Z 16 Acked 1015 Read 9 Done Done Sent false
22252023-09-22T23:24:32.019Z 17 Acked 1016 Read 6 Done Done Sent false
22262023-09-22T23:24:32.022Z 18 Acked 1017 Read 1 Done Done Sent false
22272023-09-22T23:24:32.025Z 19 Acked 1018 Read 3 Done Done Sent false
22282023-09-22T23:24:32.027Z 20 Acked 1019 Flush 0 Done Done Sent false
22292023-09-22T23:24:32.031Z 21 Acked 1020 Write 9 Done Done Sent false
22302023-09-22T23:24:32.034Z 22 Acked 1021 Write 10 Done Done Sent false
22312023-09-22T23:24:32.037Z 23 Acked 1022 Read 6 Done Done Sent false
22322023-09-22T23:24:32.040Z 24 Acked 1023 Read 9 Done Done Sent false
22332023-09-22T23:24:32.042Z 25 Acked 1024 Read 10 Done Done Sent false
22342023-09-22T23:24:32.045Z 26 Acked 1025 Write 7 Done Done Sent false
22352023-09-22T23:24:32.048Z 27 Acked 1026 Write 9 Done Done Sent false
22362023-09-22T23:24:32.050Z 28 Acked 1027 Read 4 Done Done Sent false
22372023-09-22T23:24:32.053Z 29 Acked 1028 Write 3 Done Done Sent false
22382023-09-22T23:24:32.056Z 30 Acked 1029 Read 4 Done Done Sent false
22392023-09-22T23:24:32.058Z STATES DS:0 DS:1 DS:2 TOTAL
22402023-09-22T23:24:32.061Z New 0 0 0 0
22412023-09-22T23:24:32.064Z Sent 0 0 26 26
22422023-09-22T23:24:32.067Z Done 30 30 4 64
22432023-09-22T23:24:32.070Z Skipped 0 0 0 0
22442023-09-22T23:24:32.072Z Error 0 0 0 0
22452023-09-22T23:24:32.075ZLast Flush: 1019 1019 0
22462023-09-22T23:24:32.078ZDownstairs last five completed:
22472023-09-22T23:24:32.081ZUpstairs last five completed: 30 29 28 27 26 25
22482023-09-22T23:24:32.084ZWrote out file "/tmp/test_repair_verify.data"
22492023-09-22T23:24:32.087Z
22502023-09-22T23:24:32.089ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
22512023-09-22T23:24:32.092Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.771398743Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1221}
22522023-09-22T23:24:32.095Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.77173237Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1221}
22532023-09-22T23:24:32.098Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.771756432Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1221}
22542023-09-22T23:24:32.101Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.79933482Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1221}
22552023-09-22T23:24:32.104Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.799367968Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1221}
22562023-09-22T23:24:32.107Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.799387269Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1221}
22572023-09-22T23:24:32.110Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.817267265Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1221}
22582023-09-22T23:24:32.112Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.817381661Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1221}
22592023-09-22T23:24:32.115Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:29.817466799Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1221}
22602023-09-22T23:24:32.118ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
22612023-09-22T23:24:32.120Z 0 000-019 5 5 5 7 7 7 F F F
22622023-09-22T23:24:32.123Z 1 020-039 15 15 15 23 23 23 T T F <---
22632023-09-22T23:24:32.126Z 2 040-059 12 12 12 17 17 17 F F F
22642023-09-22T23:24:32.130Z 3 060-079 14 14 14 21 21 21 F F F
22652023-09-22T23:24:32.133Z 4 080-099 16 16 14 24 24 20 F F F <---
22662023-09-22T23:24:32.136Z 5 100-119 16 16 9 24 24 14 F F F <---
22672023-09-22T23:24:32.138Z 6 120-139 16 16 15 24 24 23 F F T <---
22682023-09-22T23:24:32.141Z 7 140-159 13 13 13 19 19 19 F F F
22692023-09-22T23:24:32.144Z 8 160-179 13 13 13 19 19 19 F F F
22702023-09-22T23:24:32.147Z 9 180-199 13 13 13 19 19 19 F F F
22712023-09-22T23:24:32.149ZError: Difference in extent metadata found!
22722023-09-22T23:24:32.152Z 10 200-219 15 15 15 23 23 23 F F F
22732023-09-22T23:24:32.158Z 11 220-239 15 15 15 23 23 23 F F F
22742023-09-22T23:24:32.161Z 12 240-259 16 16 7 24 24 11 F F F <---
22752023-09-22T23:24:32.166Z 13 260-279 13 13 13 19 19 19 T T F <---
22762023-09-22T23:24:32.170Z 14 280-299 15 15 15 23 23 23 T T F <---
22772023-09-22T23:24:32.173Z 15 300-319 16 16 15 24 24 23 F F T <---
22782023-09-22T23:24:32.176Z 16 320-339 16 16 15 24 24 23 F F T <---
22792023-09-22T23:24:32.179Z 17 340-359 16 16 13 24 24 19 F F F <---
22802023-09-22T23:24:32.181Z 18 360-379 15 15 15 23 23 23 F F F
22812023-09-22T23:24:32.184Z 19 380-399 10 10 10 15 15 15 T T F <---
22822023-09-22T23:24:32.187Z 20 400-419 14 14 14 21 21 21 T T F <---
22832023-09-22T23:24:32.190Z 21 420-439 14 14 14 20 20 20 F F F
22842023-09-22T23:24:32.193Z 22 440-459 12 12 12 18 18 18 F F F
22852023-09-22T23:24:32.195Z 23 460-479 16 16 12 24 24 18 F F T <---
22862023-09-22T23:24:32.198Z 24 480-499 11 11 11 16 16 16 F F F
22872023-09-22T23:24:32.201Z 25 500-519 14 14 14 20 20 20 F F F
22882023-09-22T23:24:32.203Z 26 520-539 16 16 14 24 24 21 F F F <---
22892023-09-22T23:24:32.206Z 27 540-559 16 16 15 24 24 23 F F F <---
22902023-09-22T23:24:32.209Z 28 560-579 15 15 15 23 23 23 F F F
22912023-09-22T23:24:32.212Z 29 580-599 9 9 9 14 14 14 F F F
22922023-09-22T23:24:32.214ZMax gen: 16, Max flush: 24
22932023-09-22T23:24:32.217ZA Difference in extent metadata is expected here
22942023-09-22T23:24:32.220ZOn loop 8
22952023-09-22T23:24:32.223Z
22962023-09-22T23:24:32.225ZVerifying data now
22972023-09-22T23:24:37.534ZNo change after verify
22982023-09-22T23:24:37.537ZLoop: 8 Downstairs dump after verify (and repair):
22992023-09-22T23:24:37.552Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.448943943Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1226}
23002023-09-22T23:24:37.555Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.449293701Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1226}
23012023-09-22T23:24:37.558Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.449318402Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1226}
23022023-09-22T23:24:37.571Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.468305414Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1226}
23032023-09-22T23:24:37.574Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.468358204Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1226}
23042023-09-22T23:24:37.576Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.468379088Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1226}
23052023-09-22T23:24:37.582Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.479491298Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1226}
23062023-09-22T23:24:37.585Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.479542717Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1226}
23072023-09-22T23:24:37.587Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.479563295Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1226}
23082023-09-22T23:24:37.593ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
23092023-09-22T23:24:37.596Z 0 000-019 5 5 5 7 7 7 F F F
23102023-09-22T23:24:37.598Z 1 020-039 17 17 17 25 25 25 F F F
23112023-09-22T23:24:37.623Z 2 040-059 12 12 12 17 17 17 F F F
23122023-09-22T23:24:37.626Z 3 060-079 14 14 14 21 21 21 F F F
23132023-09-22T23:24:37.629Z 4 080-099 16 16 16 24 24 24 F F F
23142023-09-22T23:24:37.632Z 5 100-119 16 16 16 24 24 24 F F F
23152023-09-22T23:24:37.635Z 6 120-139 16 16 16 24 24 24 F F F
23162023-09-22T23:24:37.637Z 7 140-159 13 13 13 19 19 19 F F F
23172023-09-22T23:24:37.640Z 8 160-179 13 13 13 19 19 19 F F F
23182023-09-22T23:24:37.643Z 9 180-199 13 13 13 19 19 19 F F F
23192023-09-22T23:24:37.646Z 10 200-219 15 15 15 23 23 23 F F F
23202023-09-22T23:24:37.649Z 11 220-239 15 15 15 23 23 23 F F F
23212023-09-22T23:24:37.651Z 12 240-259 16 16 16 24 24 24 F F F
23222023-09-22T23:24:37.654Z 13 260-279 17 17 17 25 25 25 F F F
23232023-09-22T23:24:37.657Z 14 280-299 17 17 17 25 25 25 F F F
23242023-09-22T23:24:37.660Z 15 300-319 16 16 16 24 24 24 F F F
23252023-09-22T23:24:37.663Z 16 320-339 16 16 16 24 24 24 F F F
23262023-09-22T23:24:37.668Z 17 340-359 16 16 16 24 24 24 F F F
23272023-09-22T23:24:37.670Z 18 360-379 15 15 15 23 23 23 F F F
23282023-09-22T23:24:37.673Z 19 380-399 17 17 17 25 25 25 F F F
23292023-09-22T23:24:37.676Z 20 400-419 17 17 17 25 25 25 F F F
23302023-09-22T23:24:37.679Z 21 420-439 14 14 14 20 20 20 F F F
23312023-09-22T23:24:37.682Z 22 440-459 12 12 12 18 18 18 F F F
23322023-09-22T23:24:37.687Z 23 460-479 16 16 16 24 24 24 F F F
23332023-09-22T23:24:37.690Z 24 480-499 11 11 11 16 16 16 F F F
23342023-09-22T23:24:37.693Z 25 500-519 14 14 14 20 20 20 F F F
23352023-09-22T23:24:37.696Z 26 520-539 16 16 16 24 24 24 F F F
23362023-09-22T23:24:37.698Z 27 540-559 16 16 16 24 24 24 F F F
23372023-09-22T23:24:37.701Z 28 560-579 15 15 15 23 23 23 F F F
23382023-09-22T23:24:37.704Z 29 580-599 9 9 9 14 14 14 F F F
23392023-09-22T23:24:37.707ZMax gen: 17, Max flush: 25
23402023-09-22T23:24:37.709Z
23412023-09-22T23:24:37.712ZBegin loop 9
23422023-09-22T23:24:37.715ZDownstairs to restart: 1
23432023-09-22T23:24:37.717Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.52968281Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23442023-09-22T23:24:37.720Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.530109001Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23452023-09-22T23:24:37.723Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.530163577Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23462023-09-22T23:24:37.726Z{"msg":"Crucible stats registered with UUID: 25a44001-1b8c-406b-89df-8b0174cf3c2f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.530213344Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23472023-09-22T23:24:37.729Z{"msg":"Crucible 25a44001-1b8c-406b-89df-8b0174cf3c2f has session id: 62834875-107d-4ad7-8b78-da46d3c9d71f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.530266999Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23482023-09-22T23:24:37.731ZCrucible runtime is spawned
23492023-09-22T23:24:37.734ZThe guest has requested activation with gen:18
23502023-09-22T23:24:37.737Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.530535896Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228,"task":"up_listen"}
23512023-09-22T23:24:37.740Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.530604328Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23522023-09-22T23:24:37.744Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.530648253Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23532023-09-22T23:24:37.746Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.530670553Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228,"looper":"1"}
23542023-09-22T23:24:37.749Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.53099476Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228,"looper":"2"}
23552023-09-22T23:24:37.752Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.531241621Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228,"looper":"0"}
23562023-09-22T23:24:37.755Z{"msg":"[2] 25a44001-1b8c-406b-89df-8b0174cf3c2f looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.531342037Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228,"looper":"2"}
23572023-09-22T23:24:37.757Z{"msg":"[0] 25a44001-1b8c-406b-89df-8b0174cf3c2f looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.531434119Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228,"looper":"0"}
23582023-09-22T23:24:37.760Z{"msg":"25a44001-1b8c-406b-89df-8b0174cf3c2f active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.531498746Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23592023-09-22T23:24:37.763Z{"msg":"Set desired generation to :18","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.5315574Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23602023-09-22T23:24:37.765Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.531750796Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23612023-09-22T23:24:37.768Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.531905581Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23622023-09-22T23:24:37.771Z{"msg":"[2] 25a44001-1b8c-406b-89df-8b0174cf3c2f (e09d603a-5b45-48f7-99e4-8a85ef369a36) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.53215882Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23632023-09-22T23:24:37.774Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.532264741Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23642023-09-22T23:24:37.777Z{"msg":"[2] client is_active_req TRUE, promote! session e09d603a-5b45-48f7-99e4-8a85ef369a36","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.532288558Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23652023-09-22T23:24:37.780Z{"msg":"[0] 25a44001-1b8c-406b-89df-8b0174cf3c2f (e09d603a-5b45-48f7-99e4-8a85ef369a36) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.532416858Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23662023-09-22T23:24:37.782Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.532488191Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23672023-09-22T23:24:37.785Z{"msg":"[0] client is_active_req TRUE, promote! session e09d603a-5b45-48f7-99e4-8a85ef369a36","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.532519729Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23682023-09-22T23:24:37.787Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.532868366Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23692023-09-22T23:24:37.790Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.532962998Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23702023-09-22T23:24:37.793Z{"msg":"25a44001-1b8c-406b-89df-8b0174cf3c2f WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.533106791Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23712023-09-22T23:24:37.795Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.533166868Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23722023-09-22T23:24:37.798Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.533214828Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23732023-09-22T23:24:37.801Z{"msg":"25a44001-1b8c-406b-89df-8b0174cf3c2f WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.533255181Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23742023-09-22T23:24:37.803Z{"msg":"[2] 25a44001-1b8c-406b-89df-8b0174cf3c2f (e09d603a-5b45-48f7-99e4-8a85ef369a36) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.535759824Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23752023-09-22T23:24:37.806Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.535809676Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23762023-09-22T23:24:37.809Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:36.535882496Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23772023-09-22T23:24:37.811Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.535954189Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23782023-09-22T23:24:37.815Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.536025839Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23792023-09-22T23:24:37.817Z{"msg":"25a44001-1b8c-406b-89df-8b0174cf3c2f WaitActive New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.536100318Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23802023-09-22T23:24:37.820Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.53617089Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23812023-09-22T23:24:37.823Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.536243601Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23822023-09-22T23:24:37.826Z{"msg":"[0] 25a44001-1b8c-406b-89df-8b0174cf3c2f (e09d603a-5b45-48f7-99e4-8a85ef369a36) WaitActive New WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.536319705Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23832023-09-22T23:24:37.828Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.536346749Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23842023-09-22T23:24:37.831Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:36.536364491Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23852023-09-22T23:24:37.835Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.536381562Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23862023-09-22T23:24:37.837Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.536399262Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23872023-09-22T23:24:37.840Z{"msg":"25a44001-1b8c-406b-89df-8b0174cf3c2f WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.536417062Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23882023-09-22T23:24:37.843Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.536434521Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23892023-09-22T23:24:37.846Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:36.536452024Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23902023-09-22T23:24:38.644Z{"msg":"[1] 25a44001-1b8c-406b-89df-8b0174cf3c2f looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.53216292Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228,"looper":"1"}
23912023-09-22T23:24:38.648Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.532255944Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23922023-09-22T23:24:38.650Z{"msg":"[1] 25a44001-1b8c-406b-89df-8b0174cf3c2f (e09d603a-5b45-48f7-99e4-8a85ef369a36) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.533338296Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23932023-09-22T23:24:38.653Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.533458945Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23942023-09-22T23:24:38.656Z{"msg":"[1] client is_active_req TRUE, promote! session e09d603a-5b45-48f7-99e4-8a85ef369a36","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.533504026Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23952023-09-22T23:24:38.659Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.534107031Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23962023-09-22T23:24:38.662Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.534166031Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23972023-09-22T23:24:38.665Z{"msg":"25a44001-1b8c-406b-89df-8b0174cf3c2f WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.534191236Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23982023-09-22T23:24:38.668Z{"msg":"[1] 25a44001-1b8c-406b-89df-8b0174cf3c2f (e09d603a-5b45-48f7-99e4-8a85ef369a36) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.536584467Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
23992023-09-22T23:24:38.671Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.536636459Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24002023-09-22T23:24:38.673Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:37.536677811Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24012023-09-22T23:24:38.676Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.536717351Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24022023-09-22T23:24:38.679Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.536757121Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24032023-09-22T23:24:38.682Z{"msg":"25a44001-1b8c-406b-89df-8b0174cf3c2f WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.536797464Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24042023-09-22T23:24:38.685Z{"msg":"[0]R flush_numbers[0..12]: [7, 25, 17, 21, 24, 24, 24, 19, 19, 19, 23, 23]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.536837708Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24052023-09-22T23:24:38.688Z{"msg":"[0]R generation[0..12]: [5, 17, 12, 14, 16, 16, 16, 13, 13, 13, 15, 15]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.536859242Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24062023-09-22T23:24:38.690Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.536898093Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24072023-09-22T23:24:38.693Z{"msg":"[1]R flush_numbers[0..12]: [7, 25, 17, 21, 24, 24, 24, 19, 19, 19, 23, 23]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.536940285Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24082023-09-22T23:24:38.696Z{"msg":"[1]R generation[0..12]: [5, 17, 12, 14, 16, 16, 16, 13, 13, 13, 15, 15]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.537001412Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24092023-09-22T23:24:38.699Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0The guest has finished waiting for activation with:18
24102023-09-22T23:24:38.702ZWait for a query_work_queue command to finish before sending IO
24112023-09-22T23:24:38.705Z,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.537214157Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24122023-09-22T23:24:38.707Z{"msg":"[2]R flush_numbers[0..12]: [7, 25, 17, 21, 24, 24, 24, 19, 19, 19, 23, 23]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.537292974Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24132023-09-22T23:24:38.710Z{"msg":"[2]R generation[0..12]: [5, 17, 12, 14, 16, 16, 16, 13, 13, 13, 15, 15]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.537338972Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24142023-09-22T23:24:38.713Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.537395725Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24152023-09-22T23:24:38.716Z{"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.537464721Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24162023-09-22T23:24:38.719Z{"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.537567519Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24172023-09-22T23:24:38.722Z{"msg":"Next flush: 26","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.537624388Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24182023-09-22T23:24:38.724Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.53769281Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24192023-09-22T23:24:38.727Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.537757248Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24202023-09-22T23:24:38.730Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.537803279Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24212023-09-22T23:24:38.734Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.537869038Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24222023-09-22T23:24:38.736Z{"msg":"25a44001-1b8c-406b-89df-8b0174cf3c2f is now active with session: e09d603a-5b45-48f7-99e4-8a85ef369a36","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.53793766Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24232023-09-22T23:24:38.739Z{"msg":"25a44001-1b8c-406b-89df-8b0174cf3c2f Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.537985043Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24242023-09-22T23:24:38.742Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.538047621Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24252023-09-22T23:24:38.744Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.538104945Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24262023-09-22T23:24:38.747Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
24272023-09-22T23:24:38.750Z,"time":"2023-09-22T23:24:37.538160622Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24282023-09-22T23:24:38.753Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.538285843Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24292023-09-22T23:24:38.755Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.538353973Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24302023-09-22T23:24:38.758Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.538416247Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24312023-09-22T23:24:38.761Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.53848054Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24322023-09-22T23:24:38.764Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.538541765Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24332023-09-22T23:24:38.768Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.538602126Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24342023-09-22T23:24:38.772Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.538661893Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24352023-09-22T23:24:38.775Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.538721609Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1228}
24362023-09-22T23:24:38.777ZLoading write count information from file "/tmp/test_repair_verify.data"
24372023-09-22T23:24:38.780ZRun Repair workload
24382023-09-22T23:24:38.783Z01/30 Write block 75 len 36864 data: 2 3 3 3 4 4 7 7 5
24392023-09-22T23:24:38.785Z02/30 Read block 140 len 36864
24402023-09-22T23:24:38.788Z03/30 Read block 361 len 36864
24412023-09-22T23:24:38.791Z04/30 Read block 540 len 20480
24422023-09-22T23:24:38.794Z05/30 Flush
24432023-09-22T23:24:38.797Z06/30 Flush
24442023-09-22T23:24:38.800Z07/30 Write block 132 len 4096 data: 4
24452023-09-22T23:24:38.803Z08/30 Read block 94 len 4096
24462023-09-22T23:24:38.805Z09/30 Read block 571 len 40960
24472023-09-22T23:24:38.808Z10/30 Write block 502 len 4096 data: 3
24482023-09-22T23:24:38.811Z11/30 Write block 224 len 28672 data: 3 3 3 2 2 2 3
24492023-09-22T23:24:38.814Z12/30 Read block 295 len 12288
24502023-09-22T23:24:38.816Z13/30 Read block 168 len 16384
24512023-09-22T23:24:38.819Z14/30 Read block 7 len 8192
24522023-09-22T23:24:38.822Z15/30 Read block 3 len 28672
24532023-09-22T23:24:38.825Z16/30 Read block 43 len 20480
24542023-09-22T23:24:38.828Z17/30 Read block 19 len 8192
24552023-09-22T23:24:38.830Z18/30 Write block 538 len 36864 data: 4 4 4 3 3 3 3 2 3
24562023-09-22T23:24:38.833Z19/30 Write block 567 len 12288 data: 4 4 4
24572023-09-22T23:24:38.836Z20/30 Write block 16 len 20480 data: 2 2 2 2 3
24582023-09-22T23:24:38.839Z21/30 Write block 537 len 12288 data: 4 5 5
24592023-09-22T23:24:38.842Z22/30 Read block 375 len 8192
24602023-09-22T23:24:38.845Z23/30 Read block 174 len 16384
24612023-09-22T23:24:38.848Z24/30 Read block 214 len 24576
24622023-09-22T23:24:38.851Z25/30 Read block 559 len 16384
24632023-09-22T23:24:38.854Z26/30 Read block 141 len 8192
24642023-09-22T23:24:38.857Z27/30 Read block 568 len 8192
24652023-09-22T23:24:38.859Z28/30 Write block 69 len 12288 data: 8 6 6
24662023-09-22T23:24:38.862Z29/30 Read block 191 len 36864
24672023-09-22T23:24:38.865Z30/30 Read block 247 len 32768
24682023-09-22T23:24:38.868Z----------------------------------------------------------------
24692023-09-22T23:24:38.870Z Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:30
24702023-09-22T23:24:38.873ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
24712023-09-22T23:24:38.876Z 1 Acked 1000 Write 9 Done Done Done false
24722023-09-22T23:24:38.879Z 2 Acked 1001 Read 9 Done Done Done false
24732023-09-22T23:24:38.882Z 3 Acked 1002 Read 9 Done Sent Done false
24742023-09-22T23:24:38.884Z 4 Acked 1003 Read 5 Done Sent Done false
24752023-09-22T23:24:38.887Z 5 Acked 1004 Flush 0 Done Sent Done false
24762023-09-22T23:24:38.890Z 6 Acked 1005 Flush 0 Done Sent Done false
24772023-09-22T23:24:38.893Z 7 Acked 1006 Write 1 Done Sent Done false
24782023-09-22T23:24:38.896Z 8 Acked 1007 Read 1 Done Sent Done false
24792023-09-22T23:24:38.898Z 9 Acked 1008 Read 10 Done Sent Done false
24802023-09-22T23:24:38.901Z 10 Acked 1009 Write 1 Done Sent Done false
24812023-09-22T23:24:38.904Z 11 Acked 1010 Write 7 Done Sent Done false
24822023-09-22T23:24:38.907Z 12 Acked 1011 Read 3 Done Sent Done false
24832023-09-22T23:24:38.910Z 13 Acked 1012 Read 4 Done Sent Done false
24842023-09-22T23:24:38.913Z 14 Acked 1013 Read 2 Done Sent Done false
24852023-09-22T23:24:38.916Z 15 Acked 1014 Read 7 Done Sent Done false
24862023-09-22T23:24:38.918Z 16 Acked 1015 Read 5 Done Sent Done false
24872023-09-22T23:24:38.921Z 17 Acked 1016 Read 2 Done Sent Done false
24882023-09-22T23:24:38.924Z 18 Acked 1017 Write 9 Done Sent Done false
24892023-09-22T23:24:38.927Z 19 Acked 1018 Write 3 Done Sent Done false
24902023-09-22T23:24:38.930Z 20 Acked 1019 Write 5 Done Sent Done false
24912023-09-22T23:24:38.933Z 21 Acked 1020 Write 3 Done Sent Done false
24922023-09-22T23:24:38.936Z 22 Acked 1021 Read 2 Done Sent Done false
24932023-09-22T23:24:38.939Z 23 Acked 1022 Read 4 Done Sent Done false
24942023-09-22T23:24:38.942Z 24 Acked 1023 Read 6 Done Sent Done false
24952023-09-22T23:24:38.945Z 25 Acked 1024 Read 4 Done Sent Done false
24962023-09-22T23:24:38.948Z 26 Acked 1025 Read 2 Done Sent Done false
24972023-09-22T23:24:38.951Z 27 Acked 1026 Read 2 Done Sent Done false
24982023-09-22T23:24:38.953Z 28 Acked 1027 Write 3 Done Sent Done false
24992023-09-22T23:24:38.956Z 29 Acked 1028 Read 9 Done Sent Done false
25002023-09-22T23:24:38.959Z 30 Acked 1029 Read 8 Done Sent Done false
25012023-09-22T23:24:38.962Z STATES DS:0 DS:1 DS:2 TOTAL
25022023-09-22T23:24:38.965Z New 0 0 0 0
25032023-09-22T23:24:38.967Z Sent 0 28 0 28
25042023-09-22T23:24:38.970Z Done 30 2 30 62
25052023-09-22T23:24:38.974Z Skipped 0 0 0 0
25062023-09-22T23:24:38.976Z Error 0 0 0 0
25072023-09-22T23:24:38.979ZLast Flush: 1005 0 1005
25082023-09-22T23:24:38.981ZDownstairs last five completed:
25092023-09-22T23:24:38.984ZUpstairs last five completed: 30 29 28 27 26 25
25102023-09-22T23:24:38.987ZWrote out file "/tmp/test_repair_verify.data"
25112023-09-22T23:24:38.990Z
25122023-09-22T23:24:38.993ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
25132023-09-22T23:24:38.995Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.665082298Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1229}
25142023-09-22T23:24:38.998Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.665338138Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1229}
25152023-09-22T23:24:39.002Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.665361015Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1229}
25162023-09-22T23:24:39.005Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.698176072Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1229}
25172023-09-22T23:24:39.008Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.69821003Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1229}
25182023-09-22T23:24:39.011Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.698229887Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1229}
25192023-09-22T23:24:39.014Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.722679381Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1229}
25202023-09-22T23:24:39.017Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.722750056Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1229}
25212023-09-22T23:24:39.020Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:37.722769944Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1229}
25222023-09-22T23:24:39.028ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25232023-09-22T23:24:39.032Z 0 000-019 5 5 5 7 7 7 T F T <---
25242023-09-22T23:24:39.036Z 1 020-039 17 17 17 25 25 25 T F T <---
25252023-09-22T23:24:39.040Z 2 040-059 12 12 12 17 17 17 F F F
25262023-09-22T23:24:39.045Z 3 060-079 18 14 18 26 21 26 T T T <---
25272023-09-22T23:24:39.049ZError: Difference in extent metadata found!
25282023-09-22T23:24:39.053Z 4 080-099 18 16 18 26 24 26 F T F <---
25292023-09-22T23:24:39.055Z 5 100-119 16 16 16 24 24 24 F F F
25302023-09-22T23:24:39.058Z 6 120-139 16 16 16 24 24 24 T F T <---
25312023-09-22T23:24:39.061Z 7 140-159 13 13 13 19 19 19 F F F
25322023-09-22T23:24:39.065Z 8 160-179 13 13 13 19 19 19 F F F
25332023-09-22T23:24:39.068Z 9 180-199 13 13 13 19 19 19 F F F
25342023-09-22T23:24:39.072Z 10 200-219 15 15 15 23 23 23 F F F
25352023-09-22T23:24:39.074Z 11 220-239 15 15 15 23 23 23 T F T <---
25362023-09-22T23:24:39.078Z 12 240-259 16 16 16 24 24 24 F F F
25372023-09-22T23:24:39.081Z 13 260-279 17 17 17 25 25 25 F F F
25382023-09-22T23:24:39.083Z 14 280-299 17 17 17 25 25 25 F F F
25392023-09-22T23:24:39.087Z 15 300-319 16 16 16 24 24 24 F F F
25402023-09-22T23:24:39.090Z 16 320-339 16 16 16 24 24 24 F F F
25412023-09-22T23:24:39.093Z 17 340-359 16 16 16 24 24 24 F F F
25422023-09-22T23:24:39.096Z 18 360-379 15 15 15 23 23 23 F F F
25432023-09-22T23:24:39.099Z 19 380-399 17 17 17 25 25 25 F F F
25442023-09-22T23:24:39.102Z 20 400-419 17 17 17 25 25 25 F F F
25452023-09-22T23:24:39.104Z 21 420-439 14 14 14 20 20 20 F F F
25462023-09-22T23:24:39.107Z 22 440-459 12 12 12 18 18 18 F F F
25472023-09-22T23:24:39.110Z 23 460-479 16 16 16 24 24 24 F F F
25482023-09-22T23:24:39.113Z 24 480-499 11 11 11 16 16 16 F F F
25492023-09-22T23:24:39.116Z 25 500-519 14 14 14 20 20 20 T F T <---
25502023-09-22T23:24:39.118Z 26 520-539 16 16 16 24 24 24 T F T <---
25512023-09-22T23:24:39.121Z 27 540-559 16 16 16 24 24 24 T F T <---
25522023-09-22T23:24:39.124Z 28 560-579 15 15 15 23 23 23 T F T <---
25532023-09-22T23:24:39.127Z 29 580-599 9 9 9 14 14 14 F F F
25542023-09-22T23:24:39.130ZMax gen: 18, Max flush: 26
25552023-09-22T23:24:39.132ZA Difference in extent metadata is expected here
25562023-09-22T23:24:39.135ZOn loop 9
25572023-09-22T23:24:39.138Z
25582023-09-22T23:24:39.141ZVerifying data now
25592023-09-22T23:24:45.024ZNo change after verify
25602023-09-22T23:24:45.027ZLoop: 9 Downstairs dump after verify (and repair):
25612023-09-22T23:24:45.042Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:43.939585776Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1234}
25622023-09-22T23:24:45.045Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:43.939889207Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1234}
25632023-09-22T23:24:45.048Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:43.939913678Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1234}
25642023-09-22T23:24:45.061Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:43.958834717Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1234}
25652023-09-22T23:24:45.064Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:43.958888228Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1234}
25662023-09-22T23:24:45.067Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:43.958908432Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1234}
25672023-09-22T23:24:45.072Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:43.970030845Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1234}
25682023-09-22T23:24:45.075Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:43.970082642Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1234}
25692023-09-22T23:24:45.078Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:43.970102935Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1234}
25702023-09-22T23:24:45.083ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
25712023-09-22T23:24:45.086Z 0 000-019 19 19 19 27 27 27 F F F
25722023-09-22T23:24:45.089Z 1 020-039 19 19 19 27 27 27 F F F
25732023-09-22T23:24:45.092Z 2 040-059 12 12 12 17 17 17 F F F
25742023-09-22T23:24:45.095Z 3 060-079 19 19 19 27 27 27 F F F
25752023-09-22T23:24:45.098Z 4 080-099 18 18 18 26 26 26 F F F
25762023-09-22T23:24:45.100Z 5 100-119 16 16 16 24 24 24 F F F
25772023-09-22T23:24:45.103Z 6 120-139 19 19 19 27 27 27 F F F
25782023-09-22T23:24:45.106Z 7 140-159 13 13 13 19 19 19 F F F
25792023-09-22T23:24:45.109Z 8 160-179 13 13 13 19 19 19 F F F
25802023-09-22T23:24:45.111Z 9 180-199 13 13 13 19 19 19 F F F
25812023-09-22T23:24:45.114Z 10 200-219 15 15 15 23 23 23 F F F
25822023-09-22T23:24:45.117Z 11 220-239 19 19 19 27 27 27 F F F
25832023-09-22T23:24:45.120Z 12 240-259 16 16 16 24 24 24 F F F
25842023-09-22T23:24:45.123Z 13 260-279 17 17 17 25 25 25 F F F
25852023-09-22T23:24:45.125Z 14 280-299 17 17 17 25 25 25 F F F
25862023-09-22T23:24:45.128Z 15 300-319 16 16 16 24 24 24 F F F
25872023-09-22T23:24:45.131Z 16 320-339 16 16 16 24 24 24 F F F
25882023-09-22T23:24:45.135Z 17 340-359 16 16 16 24 24 24 F F F
25892023-09-22T23:24:45.138Z 18 360-379 15 15 15 23 23 23 F F F
25902023-09-22T23:24:45.140Z 19 380-399 17 17 17 25 25 25 F F F
25912023-09-22T23:24:45.143Z 20 400-419 17 17 17 25 25 25 F F F
25922023-09-22T23:24:45.146Z 21 420-439 14 14 14 20 20 20 F F F
25932023-09-22T23:24:45.148Z 22 440-459 12 12 12 18 18 18 F F F
25942023-09-22T23:24:45.151Z 23 460-479 16 16 16 24 24 24 F F F
25952023-09-22T23:24:45.154Z 24 480-499 11 11 11 16 16 16 F F F
25962023-09-22T23:24:45.156Z 25 500-519 19 19 19 27 27 27 F F F
25972023-09-22T23:24:45.159Z 26 520-539 19 19 19 27 27 27 F F F
25982023-09-22T23:24:45.162Z 27 540-559 19 19 19 27 27 27 F F F
25992023-09-22T23:24:45.165Z 28 560-579 19 19 19 27 27 27 F F F
26002023-09-22T23:24:45.167Z 29 580-599 9 9 9 14 14 14 F F F
26012023-09-22T23:24:45.170ZMax gen: 19, Max flush: 27
26022023-09-22T23:24:45.173Z
26032023-09-22T23:24:45.176ZBegin loop 10
26042023-09-22T23:24:45.178ZDownstairs to restart: 0
26052023-09-22T23:24:45.181Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.021029167Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26062023-09-22T23:24:45.184Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.021409043Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26072023-09-22T23:24:45.186Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.021487125Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26082023-09-22T23:24:45.189Z{"msg":"Crucible stats registered with UUID: 02c62db2-548f-4046-b4ec-861912bb49c2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.02155623Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26092023-09-22T23:24:45.192Z{"msg":"Crucible runtime is spawned
26102023-09-22T23:24:45.195ZCrucible 02c62db2-548f-4046-b4ec-861912bb49c2 has session id: 9d53daf5-75ed-4935-9650-6bb3745994db","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.021707228Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26112023-09-22T23:24:45.198Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.021778966Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236,"task":"up_listen"}
26122023-09-22T23:24:45.200ZThe guest has requested activation with gen:20
26132023-09-22T23:24:45.203Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.021842705Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26142023-09-22T23:24:45.206Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.021899426Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26152023-09-22T23:24:45.209Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.021955007Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236,"looper":"2"}
26162023-09-22T23:24:45.212Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.022240509Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236,"looper":"0"}
26172023-09-22T23:24:45.215Z{"msg":"[2] 02c62db2-548f-4046-b4ec-861912bb49c2 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.022371048Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236,"looper":"2"}
26182023-09-22T23:24:45.217Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.022441017Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236,"looper":"1"}
26192023-09-22T23:24:45.220Z{"msg":"[1] 02c62db2-548f-4046-b4ec-861912bb49c2 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.022597603Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236,"looper":"1"}
26202023-09-22T23:24:45.223Z{"msg":"02c62db2-548f-4046-b4ec-861912bb49c2 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.022863029Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26212023-09-22T23:24:45.226Z{"msg":"Set desired generation to :20","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.02291383Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26222023-09-22T23:24:45.229Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.023104619Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26232023-09-22T23:24:45.231Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.023358013Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26242023-09-22T23:24:45.234Z{"msg":"[2] 02c62db2-548f-4046-b4ec-861912bb49c2 (3f90c9d6-482e-40ea-a4fe-dedd0c408fac) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.023792321Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26252023-09-22T23:24:45.237Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.023846194Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26262023-09-22T23:24:45.240Z{"msg":"[2] client is_active_req TRUE, promote! session 3f90c9d6-482e-40ea-a4fe-dedd0c408fac","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.023904559Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26272023-09-22T23:24:45.243Z{"msg":"[1] 02c62db2-548f-4046-b4ec-861912bb49c2 (3f90c9d6-482e-40ea-a4fe-dedd0c408fac) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.024024924Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26282023-09-22T23:24:45.246Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.024119592Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26292023-09-22T23:24:45.249Z{"msg":"[1] client is_active_req TRUE, promote! session 3f90c9d6-482e-40ea-a4fe-dedd0c408fac","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.024166682Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26302023-09-22T23:24:45.253Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.024613801Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26312023-09-22T23:24:45.256Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.024701948Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26322023-09-22T23:24:45.259Z{"msg":"02c62db2-548f-4046-b4ec-861912bb49c2 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.024778994Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26332023-09-22T23:24:45.262Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.02485518Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26342023-09-22T23:24:45.265Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.024934629Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26352023-09-22T23:24:45.268Z{"msg":"02c62db2-548f-4046-b4ec-861912bb49c2 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.025007591Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26362023-09-22T23:24:45.271Z{"msg":"[1] 02c62db2-548f-4046-b4ec-861912bb49c2 (3f90c9d6-482e-40ea-a4fe-dedd0c408fac) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.026974017Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26372023-09-22T23:24:45.274Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.027042085Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26382023-09-22T23:24:45.276Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:44.027086792Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26392023-09-22T23:24:45.279Z{"msg":"[2] 02c62db2-548f-4046-b4ec-861912bb49c2 (3f90c9d6-482e-40ea-a4fe-dedd0c408fac) New WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.027128513Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26402023-09-22T23:24:45.282Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.027149506Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26412023-09-22T23:24:45.285Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:44.027187005Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26422023-09-22T23:24:45.288Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.027225345Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26432023-09-22T23:24:45.291Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.027263264Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26442023-09-22T23:24:45.294Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.027301776Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26452023-09-22T23:24:45.297Z{"msg":"02c62db2-548f-4046-b4ec-861912bb49c2 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.027340353Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26462023-09-22T23:24:45.299Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.02737882Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26472023-09-22T23:24:45.302Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.027417005Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26482023-09-22T23:24:45.305Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.027455715Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26492023-09-22T23:24:45.307Z{"msg":"02c62db2-548f-4046-b4ec-861912bb49c2 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.027493913Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26502023-09-22T23:24:45.310Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.027531939Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26512023-09-22T23:24:45.313Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:44.027569435Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26522023-09-22T23:24:46.127Z{"msg":"[0] 02c62db2-548f-4046-b4ec-861912bb49c2 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.02408405Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236,"looper":"0"}
26532023-09-22T23:24:46.134Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.024168874Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26542023-09-22T23:24:46.137Z{"msg":"[0] 02c62db2-548f-4046-b4ec-861912bb49c2 (3f90c9d6-482e-40ea-a4fe-dedd0c408fac) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.025425806Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26552023-09-22T23:24:46.140Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.02548735Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26562023-09-22T23:24:46.143Z{"msg":"[0] client is_active_req TRUE, promote! session 3f90c9d6-482e-40ea-a4fe-dedd0c408fac","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.025532254Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26572023-09-22T23:24:46.147Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.0261093Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26582023-09-22T23:24:46.150Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.026175774Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26592023-09-22T23:24:46.153Z{"msg":"02c62db2-548f-4046-b4ec-861912bb49c2 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.026219627Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26602023-09-22T23:24:46.159Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.028369889Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26612023-09-22T23:24:46.163Z{"msg":"[0] 02c62db2-548f-4046-b4ec-861912bb49c2 (3f90c9d6-482e-40ea-a4fe-dedd0c408fac) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.028515035Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26622023-09-22T23:24:46.166Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.028563462Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26632023-09-22T23:24:46.169Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:45.028609055Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26642023-09-22T23:24:46.172Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.028647396Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26652023-09-22T23:24:46.175Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.028687456Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26662023-09-22T23:24:46.177Z{"msg":"02c62db2-548f-4046-b4ec-861912bb49c2 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.02870862Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26672023-09-22T23:24:46.180Z{"msg":"[0]R flush_numbers[0..12]: [27, 27, 17, 27, 26, 24, 27, 19, 19, 19, 23, 27]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.028747991Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26682023-09-22T23:24:46.184Z{"msg":"[0]R generation[0..12]: [19, 19, 12, 19, 18, 16, 19, 13, 13, 13, 15, 19]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.028788403Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26692023-09-22T23:24:46.187Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.028809923Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26702023-09-22T23:24:46.190Z{"msg":"[1]R flush_numbers[0..12]: [27, 27, 17, 27, 26, 24, 27, 19, 19, 19, 23, 27]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.028847127Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26712023-09-22T23:24:46.193Z{"msg":"[1]R generation[0..12]: [19, 19, 12, 19, 18, 16, 19, 13, 13, 13, 15, 19]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.028885938Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26722023-09-22T23:24:46.195Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.028961532Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26732023-09-22T23:24:46.198Z{"msg":"[2]R flush_numbers[0..12]: [27, 27, 17, 27, 26, 24, 27, 19, 19, 19, 23, 27]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.028988372Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26742023-09-22T23:24:46.201Z{"msg":"[2]R generation[0..12]: [19, 19, 12, 19, 18, 16, 19, 13, 13, 13, 15, 19]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:20
26752023-09-22T23:24:46.204Z,"time":"Wait for a query_work_queue command to finish before sending IO
26762023-09-22T23:24:46.206Z2023-09-22T23:24:45.029062962Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26772023-09-22T23:24:46.209Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.029135523Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26782023-09-22T23:24:46.212Z{"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.029256747Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26792023-09-22T23:24:46.215Z{"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.029318715Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26802023-09-22T23:24:46.217Z{"msg":"Next flush: 28","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.029386312Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26812023-09-22T23:24:46.221Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.029446776Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26822023-09-22T23:24:46.224Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.029487816Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26832023-09-22T23:24:46.226Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.029542455Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26842023-09-22T23:24:46.229Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.02961076Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26852023-09-22T23:24:46.232Z{"msg":"02c62db2-548f-4046-b4ec-861912bb49c2 is now active with session: 3f90c9d6-482e-40ea-a4fe-dedd0c408fac","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.029662607Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26862023-09-22T23:24:46.235Z{"msg":"02c62db2-548f-4046-b4ec-861912bb49c2 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.029720717Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26872023-09-22T23:24:46.237Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.029788367Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26882023-09-22T23:24:46.240Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.029851939Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26892023-09-22T23:24:46.243Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.029895889Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26902023-09-22T23:24:46.246Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.02995818Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26912023-09-22T23:24:46.250Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.030016732Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26922023-09-22T23:24:46.252Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.030059145Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26932023-09-22T23:24:46.256Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
26942023-09-22T23:24:46.259Ztime":"2023-09-22T23:24:45.030098078Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26952023-09-22T23:24:46.262Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.030202046Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26962023-09-22T23:24:46.264Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.030246727Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26972023-09-22T23:24:46.267Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.030285128Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26982023-09-22T23:24:46.270Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.030322707Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1236}
26992023-09-22T23:24:46.273ZLoading write count information from file "/tmp/test_repair_verify.data"
27002023-09-22T23:24:46.276ZRun Repair workload
27012023-09-22T23:24:46.279Z01/30 Write block 170 len 8192 data: 3 3
27022023-09-22T23:24:46.282Z02/30 Read block 476 len 28672
27032023-09-22T23:24:46.284Z03/30 Write block 368 len 4096 data: 2
27042023-09-22T23:24:46.287Z04/30 Read block 149 len 24576
27052023-09-22T23:24:46.290Z05/30 Write block 189 len 28672 data: 4 4 3 2 2 2 2
27062023-09-22T23:24:46.292Z06/30 Flush
27072023-09-22T23:24:46.295Z07/30 Write block 11 len 36864 data: 2 2 2 3 3 3 3 3 3
27082023-09-22T23:24:46.298Z08/30 Read block 113 len 8192
27092023-09-22T23:24:46.301Z09/30 Write block 495 len 36864 data: 3 3 3 3 3 3 3 4 3
27102023-09-22T23:24:46.304Z10/30 Write block 424 len 4096 data: 2
27112023-09-22T23:24:46.306Z11/30 Read block 69 len 40960
27122023-09-22T23:24:46.309Z12/30 Flush
27132023-09-22T23:24:46.312Z13/30 Write block 262 len 8192 data: 2 2
27142023-09-22T23:24:46.314Z14/30 Flush
27152023-09-22T23:24:46.317Z15/30 Write block 249 len 24576 data: 3 3 2 3 2 2
27162023-09-22T23:24:46.320Z16/30 Read block 92 len 16384
27172023-09-22T23:24:46.323Z17/30 Flush
27182023-09-22T23:24:46.326Z18/30 Write block 230 len 16384 data: 4 2 2 2
27192023-09-22T23:24:46.328Z19/30 Write block 3 len 32768 data: 2 2 2 2 2 2 2 2
27202023-09-22T23:24:46.331Z20/30 Write block 577 len 32768 data: 2 2 2 3 3 3 3 3
27212023-09-22T23:24:46.334Z21/30 Read block 42 len 28672
27222023-09-22T23:24:46.337Z22/30 Read block 37 len 40960
27232023-09-22T23:24:46.340Z23/30 Read block 558 len 32768
27242023-09-22T23:24:46.343Z24/30 Write block 36 len 40960 data: 4 3 3 3 4 4 5 6 5 4
27252023-09-22T23:24:46.346Z25/30 Write block 515 len 24576 data: 3 2 2 2 2 2
27262023-09-22T23:24:46.348Z26/30 Write block 500 len 32768 data: 4 4 5 4 3 3 3 2
27272023-09-22T23:24:46.351Z27/30 Read block 159 len 8192
27282023-09-22T23:24:46.354Z28/30 Read block 293 len 40960
27292023-09-22T23:24:46.356Z29/30 Write block 537 len 12288 data: 5 6 6
27302023-09-22T23:24:46.359Z30/30 Write block 105 len 24576 data: 4 4 4 4 4 4
27312023-09-22T23:24:46.362Z----------------------------------------------------------------
27322023-09-22T23:24:46.364Z Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30
27332023-09-22T23:24:46.368ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
27342023-09-22T23:24:46.371Z 1 Acked 1000 Write 2 Done Done Done false
27352023-09-22T23:24:46.374Z 2 Acked 1001 Read 7 Done Done Done false
27362023-09-22T23:24:46.378Z 3 Acked 1002 Write 1 Done Done Done false
27372023-09-22T23:24:46.381Z 4 Acked 1003 Read 6 Done Done Done false
27382023-09-22T23:24:46.384Z 5 Acked 1004 Write 7 Done Done Done false
27392023-09-22T23:24:46.386Z 6 Acked 1005 Flush 0 Sent Done Done false
27402023-09-22T23:24:46.389Z 7 Acked 1006 Write 9 Sent Done Done false
27412023-09-22T23:24:46.392Z 8 Acked 1007 Read 2 Sent Done Done false
27422023-09-22T23:24:46.395Z 9 Acked 1008 Write 9 Sent Done Done false
27432023-09-22T23:24:46.398Z 10 Acked 1009 Write 1 Sent Done Done false
27442023-09-22T23:24:46.400Z 11 Acked 1010 Read 10 Sent Done Done false
27452023-09-22T23:24:46.403Z 12 Acked 1011 Flush 0 Sent Done Done false
27462023-09-22T23:24:46.406Z 13 Acked 1012 Write 2 Sent Done Done false
27472023-09-22T23:24:46.408Z 14 Acked 1013 Flush 0 Sent Done Done false
27482023-09-22T23:24:46.411Z 15 Acked 1014 Write 6 Sent Done Done false
27492023-09-22T23:24:46.414Z 16 Acked 1015 Read 4 Sent Done Done false
27502023-09-22T23:24:46.416Z 17 Acked 1016 Flush 0 Sent Done Done false
27512023-09-22T23:24:46.419Z 18 Acked 1017 Write 4 Sent Done Done false
27522023-09-22T23:24:46.422Z 19 Acked 1018 Write 8 Sent Done Done false
27532023-09-22T23:24:46.425Z 20 Acked 1019 Write 8 Sent Done Done false
27542023-09-22T23:24:46.427Z 21 Acked 1020 Read 7 Sent Done Done false
27552023-09-22T23:24:46.430Z 22 Acked 1021 Read 10 Sent Done Done false
27562023-09-22T23:24:46.433Z 23 Acked 1022 Read 8 Sent Done Done false
27572023-09-22T23:24:46.436Z 24 Acked 1023 Write 10 Sent Done Done false
27582023-09-22T23:24:46.438Z 25 Acked 1024 Write 6 Sent Done Done false
27592023-09-22T23:24:46.441Z 26 Acked 1025 Write 8 Sent Done Done false
27602023-09-22T23:24:46.444Z 27 Acked 1026 Read 2 Sent Done Done false
27612023-09-22T23:24:46.446Z 28 Acked 1027 Read 10 Sent Done Done false
27622023-09-22T23:24:46.449Z 29 Acked 1028 Write 3 Sent Sent Sent false
27632023-09-22T23:24:46.452Z 30 Acked 1029 Write 6 Sent Sent Sent false
27642023-09-22T23:24:46.455Z STATES DS:0 DS:1 DS:2 TOTAL
27652023-09-22T23:24:46.457Z New 0 0 0 0
27662023-09-22T23:24:46.460Z Sent 25 2 2 29
27672023-09-22T23:24:46.463Z Done 5 28 28 61
27682023-09-22T23:24:46.465Z Skipped 0 0 0 0
27692023-09-22T23:24:46.468Z Error 0 0 0 0
27702023-09-22T23:24:46.471ZLast Flush: 0 1016 1016
27712023-09-22T23:24:46.473ZDownstairs last five completed:
27722023-09-22T23:24:46.476ZUpstairs last five completed: 30 29 28 27 26 25
27732023-09-22T23:24:46.479ZWrote out file "/tmp/test_repair_verify.data"
27742023-09-22T23:24:46.482Z
27752023-09-22T23:24:46.484ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
27762023-09-22T23:24:46.487Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.178186209Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1237}
27772023-09-22T23:24:46.490Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.178550127Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1237}
27782023-09-22T23:24:46.492Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.178574368Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1237}
27792023-09-22T23:24:46.495Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.215579215Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1237}
27802023-09-22T23:24:46.498Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.215635509Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1237}
27812023-09-22T23:24:46.501Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.215674631Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1237}
27822023-09-22T23:24:46.503Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.240312172Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1237}
27832023-09-22T23:24:46.507Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.240403048Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1237}
27842023-09-22T23:24:46.510Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:45.240448366Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1237}
27852023-09-22T23:24:46.513ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
27862023-09-22T23:24:46.515Z 0 000-019 19 20 20 27 29 29 F T T <---
27872023-09-22T23:24:46.518Z 1 020-039 19 19 19 27 27 27 F T T <---
27882023-09-22T23:24:46.521Z 2 040-059 12 12 12 17 17 17 F T T <---
27892023-09-22T23:24:46.523Z 3 060-079 19 19 19 27 27 27 F F F
27902023-09-22T23:24:46.526Z 4 080-099 18 18 18 26 26 26 F F F
27912023-09-22T23:24:46.529Z 5 100-119 16 16 16 24 24 24 F T T <---
27922023-09-22T23:24:46.531Z 6 120-139 19 19 19 27 27 27 F F F
27932023-09-22T23:24:46.534Z 7 140-159 13 13 13 19 19 19 F F F
27942023-09-22T23:24:46.536Z 8 160-179 13 20 20 19 28 28 T F F <---
27952023-09-22T23:24:46.539Z 9 180-199 13 20 20 19 28 28 T F F <---
27962023-09-22T23:24:46.542Z 10 200-219 15 15 15 23 23 23 F F F
27972023-09-22T23:24:46.544Z 11 220-239 19 19 19 27 27 27 F T T <---
27982023-09-22T23:24:46.547ZError: Difference in extent metadata found!
27992023-09-22T23:24:46.549Z 12 240-259 16 20 20 24 31 31 F F F <---
28002023-09-22T23:24:46.552Z 13 260-279 17 20 20 25 30 30 F F F <---
28012023-09-22T23:24:46.555Z 14 280-299 17 17 17 25 25 25 F F F
28022023-09-22T23:24:46.557Z 15 300-319 16 16 16 24 24 24 F F F
28032023-09-22T23:24:46.560Z 16 320-339 16 16 16 24 24 24 F F F
28042023-09-22T23:24:46.563Z 17 340-359 16 16 16 24 24 24 F F F
28052023-09-22T23:24:46.567Z 18 360-379 15 20 20 23 28 28 T F F <---
28062023-09-22T23:24:46.569Z 19 380-399 17 17 17 25 25 25 F F F
28072023-09-22T23:24:46.572Z 20 400-419 17 17 17 25 25 25 F F F
28082023-09-22T23:24:46.575Z 21 420-439 14 20 20 20 29 29 F F F <---
28092023-09-22T23:24:46.577Z 22 440-459 12 12 12 18 18 18 F F F
28102023-09-22T23:24:46.580Z 23 460-479 16 16 16 24 24 24 F F F
28112023-09-22T23:24:46.583Z 24 480-499 11 20 20 16 29 29 F F F <---
28122023-09-22T23:24:46.585Z 25 500-519 19 20 20 27 29 29 F T T <---
28132023-09-22T23:24:46.588Z 26 520-539 19 19 19 27 27 27 F T T <---
28142023-09-22T23:24:46.590Z 27 540-559 19 19 19 27 27 27 F F F
28152023-09-22T23:24:46.593Z 28 560-579 19 19 19 27 27 27 F T T <---
28162023-09-22T23:24:46.596Z 29 580-599 9 9 9 14 14 14 F T T <---
28172023-09-22T23:24:46.599ZMax gen: 20, Max flush: 31
28182023-09-22T23:24:46.601ZA Difference in extent metadata is expected here
28192023-09-22T23:24:46.604ZOn loop 10
28202023-09-22T23:24:46.606Z
28212023-09-22T23:24:46.609ZVerifying data now
28222023-09-22T23:24:53.051ZNo change after verify
28232023-09-22T23:24:53.054ZLoop: 10 Downstairs dump after verify (and repair):
28242023-09-22T23:24:53.069Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:51.966277556Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1242}
28252023-09-22T23:24:53.072Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:51.966644041Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1242}
28262023-09-22T23:24:53.076Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:51.966668787Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1242}
28272023-09-22T23:24:53.088Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:51.985540274Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1242}
28282023-09-22T23:24:53.091Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:51.985595368Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1242}
28292023-09-22T23:24:53.094Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:51.985616072Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1242}
28302023-09-22T23:24:53.099Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:51.996459772Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1242}
28312023-09-22T23:24:53.101Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:51.996512091Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1242}
28322023-09-22T23:24:53.104Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:51.996533008Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1242}
28332023-09-22T23:24:53.110ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
28342023-09-22T23:24:53.112Z 0 000-019 21 21 21 32 32 32 F F F
28352023-09-22T23:24:53.115Z 1 020-039 21 21 21 32 32 32 F F F
28362023-09-22T23:24:53.119Z 2 040-059 21 21 21 32 32 32 F F F
28372023-09-22T23:24:53.122Z 3 060-079 19 19 19 27 27 27 F F F
28382023-09-22T23:24:53.124Z 4 080-099 18 18 18 26 26 26 F F F
28392023-09-22T23:24:53.127Z 5 100-119 21 21 21 32 32 32 F F F
28402023-09-22T23:24:53.130Z 6 120-139 19 19 19 27 27 27 F F F
28412023-09-22T23:24:53.133Z 7 140-159 13 13 13 19 19 19 F F F
28422023-09-22T23:24:53.136Z 8 160-179 20 20 20 28 28 28 F F F
28432023-09-22T23:24:53.138Z 9 180-199 20 20 20 28 28 28 F F F
28442023-09-22T23:24:53.141Z 10 200-219 15 15 15 23 23 23 F F F
28452023-09-22T23:24:53.144Z 11 220-239 21 21 21 32 32 32 F F F
28462023-09-22T23:24:53.147Z 12 240-259 20 20 20 31 31 31 F F F
28472023-09-22T23:24:53.150Z 13 260-279 20 20 20 30 30 30 F F F
28482023-09-22T23:24:53.153Z 14 280-299 17 17 17 25 25 25 F F F
28492023-09-22T23:24:53.156Z 15 300-319 16 16 16 24 24 24 F F F
28502023-09-22T23:24:53.159Z 16 320-339 16 16 16 24 24 24 F F F
28512023-09-22T23:24:53.161Z 17 340-359 16 16 16 24 24 24 F F F
28522023-09-22T23:24:53.164Z 18 360-379 20 20 20 28 28 28 F F F
28532023-09-22T23:24:53.167Z 19 380-399 17 17 17 25 25 25 F F F
28542023-09-22T23:24:53.170Z 20 400-419 17 17 17 25 25 25 F F F
28552023-09-22T23:24:53.172Z 21 420-439 20 20 20 29 29 29 F F F
28562023-09-22T23:24:53.175Z 22 440-459 12 12 12 18 18 18 F F F
28572023-09-22T23:24:53.177Z 23 460-479 16 16 16 24 24 24 F F F
28582023-09-22T23:24:53.180Z 24 480-499 20 20 20 29 29 29 F F F
28592023-09-22T23:24:53.183Z 25 500-519 21 21 21 32 32 32 F F F
28602023-09-22T23:24:53.185Z 26 520-539 21 21 21 32 32 32 F F F
28612023-09-22T23:24:53.188Z 27 540-559 19 19 19 27 27 27 F F F
28622023-09-22T23:24:53.191Z 28 560-579 21 21 21 32 32 32 F F F
28632023-09-22T23:24:53.194Z 29 580-599 21 21 21 32 32 32 F F F
28642023-09-22T23:24:53.196ZMax gen: 21, Max flush: 32
28652023-09-22T23:24:53.199Z
28662023-09-22T23:24:53.202ZBegin loop 11
28672023-09-22T23:24:53.205ZDownstairs to restart: 0
28682023-09-22T23:24:53.208Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.046498121Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
28692023-09-22T23:24:53.210Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.046886839Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
28702023-09-22T23:24:53.213Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.046965279Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
28712023-09-22T23:24:53.216Z{"msg":"Crucible stats registered with UUID: 1a0b3470-1f79-4b84-8209-eb863025d851","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.047008085Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
28722023-09-22T23:24:53.218ZCrucible runtime is spawned
28732023-09-22T23:24:53.221Z{"msg":"Crucible 1a0b3470-1f79-4b84-8209-eb863025d851 has session id: 9b49f563-d140-4294-9e0c-1307d7c15c48","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.047137931Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
28742023-09-22T23:24:53.224ZThe guest has requested activation with gen:22
28752023-09-22T23:24:53.226Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.047288534Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244,"task":"up_listen"}
28762023-09-22T23:24:53.229Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.04735692Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
28772023-09-22T23:24:53.232Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.047401586Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
28782023-09-22T23:24:53.235Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.047440982Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244,"looper":"0"}
28792023-09-22T23:24:53.239Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.047675579Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244,"looper":"2"}
28802023-09-22T23:24:53.241Z{"msg":"[2] 1a0b3470-1f79-4b84-8209-eb863025d851 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.047956444Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244,"looper":"2"}
28812023-09-22T23:24:53.244Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.04814553Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244,"looper":"1"}
28822023-09-22T23:24:53.248Z{"msg":"[1] 1a0b3470-1f79-4b84-8209-eb863025d851 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.048376991Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244,"looper":"1"}
28832023-09-22T23:24:53.251Z{"msg":"1a0b3470-1f79-4b84-8209-eb863025d851 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.048652392Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
28842023-09-22T23:24:53.254Z{"msg":"Set desired generation to :22","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.04872239Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
28852023-09-22T23:24:53.256Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.048898977Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
28862023-09-22T23:24:53.259Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.049264567Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
28872023-09-22T23:24:53.262Z{"msg":"[2] 1a0b3470-1f79-4b84-8209-eb863025d851 (eb9db235-ee9e-463c-bbc2-c2cf110f8381) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.049583696Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
28882023-09-22T23:24:53.264Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.049621769Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
28892023-09-22T23:24:53.267Z{"msg":"[2] client is_active_req TRUE, promote! session eb9db235-ee9e-463c-bbc2-c2cf110f8381","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.049643015Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
28902023-09-22T23:24:53.269Z{"msg":"[1] 1a0b3470-1f79-4b84-8209-eb863025d851 (eb9db235-ee9e-463c-bbc2-c2cf110f8381) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.049836882Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
28912023-09-22T23:24:53.272Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.049913914Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
28922023-09-22T23:24:53.275Z{"msg":"[1] client is_active_req TRUE, promote! session eb9db235-ee9e-463c-bbc2-c2cf110f8381","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.049966952Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
28932023-09-22T23:24:53.278Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.050415355Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
28942023-09-22T23:24:53.281Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.050523372Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
28952023-09-22T23:24:53.284Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.050675975Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
28962023-09-22T23:24:53.287Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.050765126Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
28972023-09-22T23:24:53.289Z{"msg":"1a0b3470-1f79-4b84-8209-eb863025d851 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.050829834Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
28982023-09-22T23:24:53.292Z{"msg":"1a0b3470-1f79-4b84-8209-eb863025d851 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.05087508Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
28992023-09-22T23:24:53.295Z{"msg":"[1] 1a0b3470-1f79-4b84-8209-eb863025d851 (eb9db235-ee9e-463c-bbc2-c2cf110f8381) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.052922274Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29002023-09-22T23:24:53.297Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.052980779Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29012023-09-22T23:24:53.300Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:52.053051556Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29022023-09-22T23:24:53.303Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.053113013Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29032023-09-22T23:24:53.305Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.053157128Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29042023-09-22T23:24:53.308Z{"msg":"1a0b3470-1f79-4b84-8209-eb863025d851 New WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.053196303Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29052023-09-22T23:24:53.311Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.05323468Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29062023-09-22T23:24:53.314Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.05327288Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29072023-09-22T23:24:53.317Z{"msg":"[2] 1a0b3470-1f79-4b84-8209-eb863025d851 (eb9db235-ee9e-463c-bbc2-c2cf110f8381) New WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.053940246Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29082023-09-22T23:24:53.319Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.054034866Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29092023-09-22T23:24:53.322Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:52.054080102Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29102023-09-22T23:24:53.325Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.054118552Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29112023-09-22T23:24:53.328Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.05413874Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29122023-09-22T23:24:53.331Z{"msg":"1a0b3470-1f79-4b84-8209-eb863025d851 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.054175456Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29132023-09-22T23:24:53.333Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.054213672Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29142023-09-22T23:24:53.336Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:52.05425094Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29152023-09-22T23:24:54.152Z{"msg":"[0] 1a0b3470-1f79-4b84-8209-eb863025d851 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.048948082Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244,"looper":"0"}
29162023-09-22T23:24:54.155Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.049033647Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29172023-09-22T23:24:54.158Z{"msg":"[0] 1a0b3470-1f79-4b84-8209-eb863025d851 (eb9db235-ee9e-463c-bbc2-c2cf110f8381) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.050109146Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29182023-09-22T23:24:54.161Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.050175261Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29192023-09-22T23:24:54.164Z{"msg":"[0] client is_active_req TRUE, promote! session eb9db235-ee9e-463c-bbc2-c2cf110f8381","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.05019943Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29202023-09-22T23:24:54.166Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.050813822Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29212023-09-22T23:24:54.169Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.050874338Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29222023-09-22T23:24:54.172Z{"msg":"1a0b3470-1f79-4b84-8209-eb863025d851 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.050917103Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29232023-09-22T23:24:54.175Z{"msg":"[0] 1a0b3470-1f79-4b84-8209-eb863025d851 (eb9db235-ee9e-463c-bbc2-c2cf110f8381) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.05333689Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29242023-09-22T23:24:54.177Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.053454224Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29252023-09-22T23:24:54.180Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:53.053513218Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29262023-09-22T23:24:54.183Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.053562218Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29272023-09-22T23:24:54.186Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.053602308Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29282023-09-22T23:24:54.189Z{"msg":"1a0b3470-1f79-4b84-8209-eb863025d851 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.053642901Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29292023-09-22T23:24:54.192Z{"msg":"[0]R flush_numbers[0..12]: [32, 32, 32, 27, 26, 32, 27, 19, 28, 28, 23, 32]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.053682993Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29302023-09-22T23:24:54.195ZThe guest has finished waiting for activation with:22
29312023-09-22T23:24:54.198Z{"msg":"[0]R generation[0..12]: [21, 21, 21, 19, 18, 21, 19, 13, 20, 20, 15, 21]","v":0,"nameWait for a query_work_queue command to finish before sending IO
29322023-09-22T23:24:54.201Z":"crucible","level":30,"time":"2023-09-22T23:24:53.053738577Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29332023-09-22T23:24:54.204Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.053786138Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29342023-09-22T23:24:54.206Z{"msg":"[1]R flush_numbers[0..12]: [32, 32, 32, 27, 26, 32, 27, 19, 28, 28, 23, 32]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.05380797Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29352023-09-22T23:24:54.209Z{"msg":"[1]R generation[0..12]: [21, 21, 21, 19, 18, 21, 19, 13, 20, 20, 15, 21]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.053852499Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29362023-09-22T23:24:54.212Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.05389956Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29372023-09-22T23:24:54.215Z{"msg":"[2]R flush_numbers[0..12]: [32, 32, 32, 27, 26, 32, 27, 19, 28, 28, 23, 32]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.053924689Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29382023-09-22T23:24:54.218Z{"msg":"[2]R generation[0..12]: [21, 21, 21, 19, 18, 21, 19, 13, 20, 20, 15, 21]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.053963471Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29392023-09-22T23:24:54.221Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.053989971Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29402023-09-22T23:24:54.224Z{"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.054045092Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29412023-09-22T23:24:54.227Z{"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.05413998Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29422023-09-22T23:24:54.230Z{"msg":"Next flush: 33","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.054161145Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29432023-09-22T23:24:54.232Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.054203976Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29442023-09-22T23:24:54.235Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.054242361Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29452023-09-22T23:24:54.238Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.054287572Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29462023-09-22T23:24:54.241Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.054335732Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29472023-09-22T23:24:54.250Z{"msg":"1a0b3470-1f79-4b84-8209-eb863025d851 is now active with session: eb9db235-ee9e-463c-bbc2-c2cf110f8381","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.054375948Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29482023-09-22T23:24:54.259Z{"msg":"1a0b3470-1f79-4b84-8209-eb863025d851 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.054414911Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29492023-09-22T23:24:54.263Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.054453191Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29502023-09-22T23:24:54.265Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.054492237Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29512023-09-22T23:24:54.268ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
29522023-09-22T23:24:54.271Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.054535499Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29532023-09-22T23:24:54.274Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.054573328Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29542023-09-22T23:24:54.276Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.05461051Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29552023-09-22T23:24:54.279Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.054648091Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29562023-09-22T23:24:54.282Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.054685118Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29572023-09-22T23:24:54.285Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.054703973Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29582023-09-22T23:24:54.287Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.054721087Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29592023-09-22T23:24:54.290Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.054756677Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29602023-09-22T23:24:54.293Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.05479371Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1244}
29612023-09-22T23:24:54.295ZLoading write count information from file "/tmp/test_repair_verify.data"
29622023-09-22T23:24:54.298ZRun Repair workload
29632023-09-22T23:24:54.301Z01/30 Flush
29642023-09-22T23:24:54.303Z02/30 Write block 239 len 20480 data: 2 2 2 2 3
29652023-09-22T23:24:54.306Z03/30 Read block 366 len 8192
29662023-09-22T23:24:54.309Z04/30 Write block 160 len 40960 data: 4 4 3 2 2 2 2 2 2 2
29672023-09-22T23:24:54.312Z05/30 Read block 394 len 36864
29682023-09-22T23:24:54.315Z06/30 Write block 377 len 16384 data: 2 2 2 3
29692023-09-22T23:24:54.318Z07/30 Read block 560 len 8192
29702023-09-22T23:24:54.320Z08/30 Read block 184 len 16384
29712023-09-22T23:24:54.323Z09/30 Read block 556 len 36864
29722023-09-22T23:24:54.326Z10/30 Write block 514 len 20480 data: 4 4 3 3 3
29732023-09-22T23:24:54.330Z11/30 Read block 322 len 36864
29742023-09-22T23:24:54.333Z12/30 Write block 61 len 36864 data: 4 4 5 5 5 5 7 8 9
29752023-09-22T23:24:54.336Z13/30 Write block 481 len 20480 data: 2 2 2 2 2
29762023-09-22T23:24:54.338Z14/30 Write block 484 len 28672 data: 3 3 3 3 4 5 4
29772023-09-22T23:24:54.342Z15/30 Read block 213 len 28672
29782023-09-22T23:24:54.345Z16/30 Read block 226 len 36864
29792023-09-22T23:24:54.348Z17/30 Read block 90 len 16384
29802023-09-22T23:24:54.350Z18/30 Read block 174 len 40960
29812023-09-22T23:24:54.354Z19/30 Write block 132 len 40960 data: 5 3 2 2 3 2 2 2 3 3
29822023-09-22T23:24:54.356Z20/30 Write block 567 len 12288 data: 5 5 5
29832023-09-22T23:24:54.359Z21/30 Read block 191 len 8192
29842023-09-22T23:24:54.362Z22/30 Read block 489 len 12288
29852023-09-22T23:24:54.364Z23/30 Write block 47 len 24576 data: 4 4 4 3 3 2
29862023-09-22T23:24:54.367Z24/30 Read block 149 len 8192
29872023-09-22T23:24:54.370Z25/30 Read block 89 len 20480
29882023-09-22T23:24:54.373Z26/30 Read block 197 len 32768
29892023-09-22T23:24:54.375Z27/30 Write block 2 len 32768 data: 2 3 3 3 3 3 3 3
29902023-09-22T23:24:54.378Z28/30 Read block 220 len 28672
29912023-09-22T23:24:54.381Z29/30 Write block 448 len 4096 data: 3
29922023-09-22T23:24:54.384Z30/30 Read block 337 len 20480
29932023-09-22T23:24:54.387Z----------------------------------------------------------------
29942023-09-22T23:24:54.390Z Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:29
29952023-09-22T23:24:54.393ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
29962023-09-22T23:24:54.395Z 2 Acked 1001 Write 5 Done Done Done false
29972023-09-22T23:24:54.398Z 3 Acked 1002 Read 2 Done Done Done false
29982023-09-22T23:24:54.401Z 4 Acked 1003 Write 10 Done Done Done false
29992023-09-22T23:24:54.404Z 5 Acked 1004 Read 9 Done Done Done false
30002023-09-22T23:24:54.407Z 6 Acked 1005 Write 4 Done Done Done false
30012023-09-22T23:24:54.410Z 7 Acked 1006 Read 2 Done Done Done false
30022023-09-22T23:24:54.413Z 8 Acked 1007 Read 4 Done Done Done false
30032023-09-22T23:24:54.416Z 9 Acked 1008 Read 9 Sent Done Done false
30042023-09-22T23:24:54.418Z 10 Acked 1009 Write 5 Sent Done Done false
30052023-09-22T23:24:54.421Z 11 Acked 1010 Read 9 Sent Done Done false
30062023-09-22T23:24:54.424Z 12 Acked 1011 Write 9 Sent Done Done false
30072023-09-22T23:24:54.427Z 13 Acked 1012 Write 5 Sent Done Done false
30082023-09-22T23:24:54.430Z 14 Acked 1013 Write 7 Sent Done Done false
30092023-09-22T23:24:54.433Z 15 Acked 1014 Read 7 Sent Done Done false
30102023-09-22T23:24:54.436Z 16 Acked 1015 Read 9 Sent Done Done false
30112023-09-22T23:24:54.439Z 17 Acked 1016 Read 4 Sent Done Done false
30122023-09-22T23:24:54.442Z 18 Acked 1017 Read 10 Sent Done Done false
30132023-09-22T23:24:54.445Z 19 Acked 1018 Write 10 Sent Done Done false
30142023-09-22T23:24:54.448Z 20 Acked 1019 Write 3 Sent Done Done false
30152023-09-22T23:24:54.451Z 21 Acked 1020 Read 2 Sent Done Done false
30162023-09-22T23:24:54.453Z 22 Acked 1021 Read 3 Sent Done Done false
30172023-09-22T23:24:54.456Z 23 Acked 1022 Write 6 Sent Done Done false
30182023-09-22T23:24:54.459Z 24 Acked 1023 Read 2 Sent Done Done false
30192023-09-22T23:24:54.462Z 25 Acked 1024 Read 5 Sent Done Done false
30202023-09-22T23:24:54.464Z 26 Acked 1025 Read 8 Sent Done Done false
30212023-09-22T23:24:54.468Z 27 Acked 1026 Write 8 Sent Done Done false
30222023-09-22T23:24:54.470Z 28 Acked 1027 Read 7 Sent Done Done false
30232023-09-22T23:24:54.473Z 29 Acked 1028 Write 1 Sent Done Done false
30242023-09-22T23:24:54.476Z 30 Acked 1029 Read 5 Sent Done Done false
30252023-09-22T23:24:54.478Z STATES DS:0 DS:1 DS:2 TOTAL
30262023-09-22T23:24:54.481Z New 0 0 0 0
30272023-09-22T23:24:54.484Z Sent 22 0 0 22
30282023-09-22T23:24:54.486Z Done 7 29 29 65
30292023-09-22T23:24:54.489Z Skipped 0 0 0 0
30302023-09-22T23:24:54.491Z Error 0 0 0 0
30312023-09-22T23:24:54.494ZLast Flush: 1000 1000 1000
30322023-09-22T23:24:54.497ZDownstairs last five completed: 1000
30332023-09-22T23:24:54.500ZUpstairs last five completed: 30 29 28 27 26 25
30342023-09-22T23:24:54.503ZWrote out file "/tmp/test_repair_verify.data"
30352023-09-22T23:24:54.505Z
30362023-09-22T23:24:54.508ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
30372023-09-22T23:24:54.511Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.195527049Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1245}
30382023-09-22T23:24:54.514Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.195939742Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1245}
30392023-09-22T23:24:54.517Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.195981404Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1245}
30402023-09-22T23:24:54.519Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.241694582Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1245}
30412023-09-22T23:24:54.522Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.241760314Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1245}
30422023-09-22T23:24:54.526Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.24179959Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1245}
30432023-09-22T23:24:54.529Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.273507233Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1245}
30442023-09-22T23:24:54.532Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.273572788Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1245}
30452023-09-22T23:24:54.534Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:53.273596614Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1245}
30462023-09-22T23:24:54.537ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
30472023-09-22T23:24:54.540Z 0 000-019 21 21 21 32 32 32 F T T <---
30482023-09-22T23:24:54.544Z 1 020-039 21 21 21 32 32 32 F F F
30492023-09-22T23:24:54.548Z 2 040-059 21 21 21 32 32 32 F T T <---
30502023-09-22T23:24:54.551Z 3 060-079 19 19 19 27 27 27 F T T <---
30512023-09-22T23:24:54.554Z 4 080-099 18 18 18 26 26 26 F F F
30522023-09-22T23:24:54.556Z 5 100-119 21 21 21 32 32 32 F F F
30532023-09-22T23:24:54.559Z 6 120-139 19 19 19 27 27 27 F T T <---
30542023-09-22T23:24:54.561Z 7 140-159 13 13 13 19 19 19 F T T <---
30552023-09-22T23:24:54.564Z 8 160-179 20 20 20 28 28 28 T T T
30562023-09-22T23:24:54.567Z 9 180-199 20 20 20 28 28 28 F F F
30572023-09-22T23:24:54.570Z 10 200-219 15 15 15 23 23 23 F F F
30582023-09-22T23:24:54.572Z 11 220-239 21 21 21 32 32 32 T T T
30592023-09-22T23:24:54.575Z 12 240-259 20 20 20 31 31 31 T T T
30602023-09-22T23:24:54.578Z 13 260-279 20 20 20 30 30 30 F F F
30612023-09-22T23:24:54.580Z 14 280-299 17 17 17 25 25 25 F F F
30622023-09-22T23:24:54.583Z 15 300-319 16 16 16 24 24 24 F F F
30632023-09-22T23:24:54.586Z 16 320-339 16 16 16 24 24 24 F F F
30642023-09-22T23:24:54.589Z 17 340-359 16 16 16 24 24 24 F F F
30652023-09-22T23:24:54.592Z 18 360-379 20 20 20 28 28 28 T T T
30662023-09-22T23:24:54.594ZError: Difference in extent metadata found!
30672023-09-22T23:24:54.597Z 19 380-399 17 17 17 25 25 25 T T T
30682023-09-22T23:24:54.600Z 20 400-419 17 17 17 25 25 25 F F F
30692023-09-22T23:24:54.602Z 21 420-439 20 20 20 29 29 29 F F F
30702023-09-22T23:24:54.606Z 22 440-459 12 12 12 18 18 18 F T T <---
30712023-09-22T23:24:54.609Z 23 460-479 16 16 16 24 24 24 F F F
30722023-09-22T23:24:54.612Z 24 480-499 20 20 20 29 29 29 F T T <---
30732023-09-22T23:24:54.614Z 25 500-519 21 21 21 32 32 32 F T T <---
30742023-09-22T23:24:54.617Z 26 520-539 21 21 21 32 32 32 F F F
30752023-09-22T23:24:54.620Z 27 540-559 19 19 19 27 27 27 F F F
30762023-09-22T23:24:54.623Z 28 560-579 21 21 21 32 32 32 F T T <---
30772023-09-22T23:24:54.626Z 29 580-599 21 21 21 32 32 32 F F F
30782023-09-22T23:24:54.628ZMax gen: 21, Max flush: 32
30792023-09-22T23:24:54.631ZA Difference in extent metadata is expected here
30802023-09-22T23:24:54.634ZOn loop 11
30812023-09-22T23:24:54.637Z
30822023-09-22T23:24:54.640ZVerifying data now
30832023-09-22T23:25:00.973ZNo change after verify
30842023-09-22T23:25:00.976ZLoop: 11 Downstairs dump after verify (and repair):
30852023-09-22T23:25:00.991Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.887928638Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1250}
30862023-09-22T23:25:00.994Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.888255307Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1250}
30872023-09-22T23:25:00.997Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.888279866Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1250}
30882023-09-22T23:25:01.010Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.907380941Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1250}
30892023-09-22T23:25:01.012Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.907434334Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1250}
30902023-09-22T23:25:01.015Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.907455079Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1250}
30912023-09-22T23:25:01.021Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.918586227Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1250}
30922023-09-22T23:25:01.024Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.91863815Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1250}
30932023-09-22T23:25:01.027Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.918658699Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1250}
30942023-09-22T23:25:01.032ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
30952023-09-22T23:25:01.035Z 0 000-019 22 22 22 33 33 33 F F F
30962023-09-22T23:25:01.038Z 1 020-039 21 21 21 32 32 32 F F F
30972023-09-22T23:25:01.041Z 2 040-059 22 22 22 33 33 33 F F F
30982023-09-22T23:25:01.044Z 3 060-079 22 22 22 33 33 33 F F F
30992023-09-22T23:25:01.047Z 4 080-099 18 18 18 26 26 26 F F F
31002023-09-22T23:25:01.050Z 5 100-119 21 21 21 32 32 32 F F F
31012023-09-22T23:25:01.053Z 6 120-139 22 22 22 33 33 33 F F F
31022023-09-22T23:25:01.056Z 7 140-159 22 22 22 33 33 33 F F F
31032023-09-22T23:25:01.059Z 8 160-179 22 22 22 33 33 33 F F F
31042023-09-22T23:25:01.062Z 9 180-199 20 20 20 28 28 28 F F F
31052023-09-22T23:25:01.065Z 10 200-219 15 15 15 23 23 23 F F F
31062023-09-22T23:25:01.068Z 11 220-239 22 22 22 33 33 33 F F F
31072023-09-22T23:25:01.070Z 12 240-259 22 22 22 33 33 33 F F F
31082023-09-22T23:25:01.073Z 13 260-279 20 20 20 30 30 30 F F F
31092023-09-22T23:25:01.076Z 14 280-299 17 17 17 25 25 25 F F F
31102023-09-22T23:25:01.079Z 15 300-319 16 16 16 24 24 24 F F F
31112023-09-22T23:25:01.082Z 16 320-339 16 16 16 24 24 24 F F F
31122023-09-22T23:25:01.085Z 17 340-359 16 16 16 24 24 24 F F F
31132023-09-22T23:25:01.088Z 18 360-379 22 22 22 33 33 33 F F F
31142023-09-22T23:25:01.091Z 19 380-399 22 22 22 33 33 33 F F F
31152023-09-22T23:25:01.093Z 20 400-419 17 17 17 25 25 25 F F F
31162023-09-22T23:25:01.096Z 21 420-439 20 20 20 29 29 29 F F F
31172023-09-22T23:25:01.099Z 22 440-459 22 22 22 33 33 33 F F F
31182023-09-22T23:25:01.122Z 23 460-479 16 16 16 24 24 24 F F F
31192023-09-22T23:25:01.125Z 24 480-499 22 22 22 33 33 33 F F F
31202023-09-22T23:25:01.128Z 25 500-519 22 22 22 33 33 33 F F F
31212023-09-22T23:25:01.131Z 26 520-539 21 21 21 32 32 32 F F F
31222023-09-22T23:25:01.134Z 27 540-559 19 19 19 27 27 27 F F F
31232023-09-22T23:25:01.137Z 28 560-579 22 22 22 33 33 33 F F F
31242023-09-22T23:25:01.140Z 29 580-599 21 21 21 32 32 32 F F F
31252023-09-22T23:25:01.143ZMax gen: 22, Max flush: 33
31262023-09-22T23:25:01.146Z
31272023-09-22T23:25:01.149ZBegin loop 12
31282023-09-22T23:25:01.152ZDownstairs to restart: 2
31292023-09-22T23:25:01.155Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.969622777Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31302023-09-22T23:25:01.158Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.96999143Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31312023-09-22T23:25:01.162Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.970061823Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31322023-09-22T23:25:01.165Z{"msg":"Crucible stats registered with UUID: d6eb3908-f06d-40a2-a124-06b431bceae8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.970113787Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31332023-09-22T23:25:01.167Z{"msg":"Crucible d6eb3908-f06d-40a2-a124-06b431bceae8 has session id: fa6e1b9f-7746-4f69-a56a-ded3b0ed83cc","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.970181219Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31342023-09-22T23:25:01.171ZCrucible runtime is spawned
31352023-09-22T23:25:01.174Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.970376328Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252,"task":"up_listen"}
31362023-09-22T23:25:01.177Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.970452186Z","hostname":"The guest has requested activation with gen:24
31372023-09-22T23:25:01.180Zip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31382023-09-22T23:25:01.183Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.970587475Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31392023-09-22T23:25:01.186Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.970657663Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252,"looper":"1"}
31402023-09-22T23:25:01.189Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.970918007Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252,"looper":"0"}
31412023-09-22T23:25:01.192Z{"msg":"[1] d6eb3908-f06d-40a2-a124-06b431bceae8 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.971093284Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252,"looper":"1"}
31422023-09-22T23:25:01.195Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.971221674Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252,"looper":"2"}
31432023-09-22T23:25:01.198Z{"msg":"[0] d6eb3908-f06d-40a2-a124-06b431bceae8 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.971350247Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252,"looper":"0"}
31442023-09-22T23:25:01.201Z{"msg":"d6eb3908-f06d-40a2-a124-06b431bceae8 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.971735953Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31452023-09-22T23:25:01.204Z{"msg":"Set desired generation to :24","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.971788753Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31462023-09-22T23:25:01.207Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.971840399Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31472023-09-22T23:25:01.210Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.972164097Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31482023-09-22T23:25:01.213Z{"msg":"[1] d6eb3908-f06d-40a2-a124-06b431bceae8 (f7a5857e-bbd1-4315-b726-3f114aad3b39) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.972527729Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31492023-09-22T23:25:01.215Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.972590199Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31502023-09-22T23:25:01.218Z{"msg":"[1] client is_active_req TRUE, promote! session f7a5857e-bbd1-4315-b726-3f114aad3b39","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.972633597Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31512023-09-22T23:25:01.221Z{"msg":"[0] d6eb3908-f06d-40a2-a124-06b431bceae8 (f7a5857e-bbd1-4315-b726-3f114aad3b39) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.972749689Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31522023-09-22T23:25:01.224Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.972823323Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31532023-09-22T23:25:01.227Z{"msg":"[0] client is_active_req TRUE, promote! session f7a5857e-bbd1-4315-b726-3f114aad3b39","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.97288727Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31542023-09-22T23:25:01.232Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.973352891Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31552023-09-22T23:25:01.235Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.973409074Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31562023-09-22T23:25:01.237Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.973444429Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31572023-09-22T23:25:01.240Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.973519488Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31582023-09-22T23:25:01.243Z{"msg":"d6eb3908-f06d-40a2-a124-06b431bceae8 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.97363693Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31592023-09-22T23:25:01.246Z{"msg":"d6eb3908-f06d-40a2-a124-06b431bceae8 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.973742194Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31602023-09-22T23:25:01.249Z{"msg":"[1] d6eb3908-f06d-40a2-a124-06b431bceae8 (f7a5857e-bbd1-4315-b726-3f114aad3b39) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.975578127Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31612023-09-22T23:25:01.252Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.975659915Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31622023-09-22T23:25:01.255Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:59.97573136Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31632023-09-22T23:25:01.258Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.975802846Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31642023-09-22T23:25:01.261Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.975876131Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31652023-09-22T23:25:01.264Z{"msg":"d6eb3908-f06d-40a2-a124-06b431bceae8 WaitActive WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.975952172Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31662023-09-22T23:25:01.267Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.975991204Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31672023-09-22T23:25:01.270Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.976060361Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31682023-09-22T23:25:01.273Z{"msg":"[0] d6eb3908-f06d-40a2-a124-06b431bceae8 (f7a5857e-bbd1-4315-b726-3f114aad3b39) WaitActive WaitQuorum New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.976138224Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31692023-09-22T23:25:01.276Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.976187722Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31702023-09-22T23:25:01.279Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:24:59.97622684Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31712023-09-22T23:25:01.283Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.97626462Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31722023-09-22T23:25:01.286Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.976302873Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31732023-09-22T23:25:01.288Z{"msg":"d6eb3908-f06d-40a2-a124-06b431bceae8 WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.976340904Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31742023-09-22T23:25:01.292Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.976378554Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31752023-09-22T23:25:01.296Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:24:59.97641547Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31762023-09-22T23:25:02.076Z{"msg":"[2] d6eb3908-f06d-40a2-a124-06b431bceae8 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.973181966Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252,"looper":"2"}
31772023-09-22T23:25:02.080Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.973265873Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31782023-09-22T23:25:02.086Z{"msg":"[2] d6eb3908-f06d-40a2-a124-06b431bceae8 (f7a5857e-bbd1-4315-b726-3f114aad3b39) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.974329283Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31792023-09-22T23:25:02.089Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.974389763Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31802023-09-22T23:25:02.093Z{"msg":"[2] client is_active_req TRUE, promote! session f7a5857e-bbd1-4315-b726-3f114aad3b39","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.974433402Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31812023-09-22T23:25:02.096Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.975041147Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31822023-09-22T23:25:02.099Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.975100015Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31832023-09-22T23:25:02.103Z{"msg":"d6eb3908-f06d-40a2-a124-06b431bceae8 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.975143073Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31842023-09-22T23:25:02.106Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.977298971Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31852023-09-22T23:25:02.109Z{"msg":"[2] d6eb3908-f06d-40a2-a124-06b431bceae8 (f7a5857e-bbd1-4315-b726-3f114aad3b39) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.977577794Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31862023-09-22T23:25:02.113Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.977729967Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31872023-09-22T23:25:02.116Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:00.977775255Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31882023-09-22T23:25:02.119Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.977815013Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31892023-09-22T23:25:02.143Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.977867255Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31902023-09-22T23:25:02.148ZThe guest has finished waiting for activation with:24
31912023-09-22T23:25:02.151Z{"msg":"Wait for a query_work_queue command to finish before sending IO
31922023-09-22T23:25:02.154Zd6eb3908-f06d-40a2-a124-06b431bceae8 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.977938581Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31932023-09-22T23:25:02.157Z{"msg":"[0]R flush_numbers[0..12]: [33, 32, 33, 33, 26, 32, 33, 33, 33, 28, 23, 33]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.97805432Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31942023-09-22T23:25:02.161Z{"msg":"[0]R generation[0..12]: [22, 21, 22, 22, 18, 21, 22, 22, 22, 20, 15, 22]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.978121277Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31952023-09-22T23:25:02.165Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.978210116Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31962023-09-22T23:25:02.168Z{"msg":"[1]R flush_numbers[0..12]: [33, 32, 33, 33, 26, 32, 33, 33, 33, 28, 23, 33]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.978237227Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31972023-09-22T23:25:02.171Z{"msg":"[1]R generation[0..12]: [22, 21, 22, 22, 18, 21, 22, 22, 22, 20, 15, 22]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.97828632Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31982023-09-22T23:25:02.174Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.978308073Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
31992023-09-22T23:25:02.177Z{"msg":"[2]R flush_numbers[0..12]: [33, 32, 33, 33, 26, 32, 33, 33, 33, 28, 23, 33]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.978354273Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
32002023-09-22T23:25:02.180Z{"msg":"[2]R generation[0..12]: [22, 21, 22, 22, 18, 21, 22, 22, 22, 20, 15, 22]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.978374792Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
32012023-09-22T23:25:02.183Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.978412709Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
32022023-09-22T23:25:02.186Z{"msg":"Max found gen is 23","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.978455528Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
32032023-09-22T23:25:02.189Z{"msg":"Generation requested: 24 >= found:23","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.978502453Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
32042023-09-22T23:25:02.192Z{"msg":"Next flush: 34","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.978541872Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
32052023-09-22T23:25:02.195Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.97857965Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
32062023-09-22T23:25:02.198Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.978617632Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
32072023-09-22T23:25:02.201Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.978656214Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
32082023-09-22T23:25:02.204Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.978702851Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
32092023-09-22T23:25:02.207Z{"msg":"d6eb3908-f06d-40a2-a124-06b431bceae8 is now active with session: f7a5857e-bbd1-4315-b726-3f114aad3b39","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.978746974Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
32102023-09-22T23:25:02.210Z{"msg":"d6eb3908-f06d-40a2-a124-06b431bceae8 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.978785713Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
32112023-09-22T23:25:02.213Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.978835472Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
32122023-09-22T23:25:02.216Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.978881288Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
32132023-09-22T23:25:02.220Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.978927148Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
32142023-09-22T23:25:02.222Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.978968353Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
32152023-09-22T23:25:02.226Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.978988836Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
32162023-09-22T23:25:02.228Z1252}
32172023-09-22T23:25:02.231Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.979089034Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
32182023-09-22T23:25:02.234Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.979134144Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
32192023-09-22T23:25:02.237Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.979172173Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
32202023-09-22T23:25:02.241Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.97920993Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
32212023-09-22T23:25:02.244Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.979247656Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
32222023-09-22T23:25:02.247Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:00.979285054Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1252}
32232023-09-22T23:25:02.250ZLoading write count information from file "/tmp/test_repair_verify.data"
32242023-09-22T23:25:02.253ZRun Repair workload
32252023-09-22T23:25:02.256Z01/30 Write block 488 len 16384 data: 5 6 5 4
32262023-09-22T23:25:02.259Z02/30 Write block 36 len 24576 data: 5 4 4 4 5 5
32272023-09-22T23:25:02.262Z03/30 Read block 103 len 32768
32282023-09-22T23:25:02.265Z04/30 Read block 158 len 16384
32292023-09-22T23:25:02.268Z05/30 Write block 167 len 12288 data: 3 3 3
32302023-09-22T23:25:02.271Z06/30 Write block 37 len 20480 data: 5 5 5 6 6
32312023-09-22T23:25:02.274Z07/30 Read block 234 len 24576
32322023-09-22T23:25:02.277Z08/30 Write block 493 len 8192 data: 4 3
32332023-09-22T23:25:02.281Z09/30 Read block 346 len 28672
32342023-09-22T23:25:02.283Z10/30 Write block 202 len 40960 data: 3 3 3 4 5 5 5 4 4 5
32352023-09-22T23:25:02.286Z11/30 Read block 496 len 16384
32362023-09-22T23:25:02.289Z12/30 Read block 123 len 24576
32372023-09-22T23:25:02.292Z13/30 Write block 182 len 4096 data: 3
32382023-09-22T23:25:02.295Z14/30 Write block 474 len 4096 data: 2
32392023-09-22T23:25:02.304Z15/30 Write block 215 len 28672 data: 4 3 3 3 3 4 3
32402023-09-22T23:25:02.314Z16/30 Write block 35 len 24576 data: 3 6 6 6 6 7
32412023-09-22T23:25:02.318Z17/30 Write block 444 len 12288 data: 3 3 3
32422023-09-22T23:25:02.321Z18/30 Read block 339 len 4096
32432023-09-22T23:25:02.324Z19/30 Flush
32442023-09-22T23:25:02.327Z20/30 Write block 130 len 32768 data: 3 3 6 4 3 3 4 3
32452023-09-22T23:25:02.330Z21/30 Write block 509 len 4096 data: 3
32462023-09-22T23:25:02.333Z22/30 Read block 387 len 16384
32472023-09-22T23:25:02.336Z23/30 Write block 347 len 36864 data: 5 4 2 2 2 2 2 2 3
32482023-09-22T23:25:02.339Z24/30 Write block 386 len 28672 data: 3 5 4 3 3 3 3
32492023-09-22T23:25:02.342Z25/30 Write block 444 len 12288 data: 4 4 4
32502023-09-22T23:25:02.345Z26/30 Flush
32512023-09-22T23:25:02.348Z27/30 Write block 261 len 28672 data: 2 3 3 2 2 2 3
32522023-09-22T23:25:02.351Z28/30 Write block 120 len 24576 data: 3 3 4 4 4 4
32532023-09-22T23:25:02.354Z29/30 Write block 334 len 24576 data: 3 3 3 3 3 3
32542023-09-22T23:25:02.357Z30/30 Write block 432 len 16384 data: 4 3 2 2
32552023-09-22T23:25:02.360Z----------------------------------------------------------------
32562023-09-22T23:25:02.363Z Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:30
32572023-09-22T23:25:02.366ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
32582023-09-22T23:25:02.370Z 1 Acked 1000 Write 4 Done Done Done false
32592023-09-22T23:25:02.373Z 2 Acked 1001 Write 6 Done Done Done false
32602023-09-22T23:25:02.376Z 3 Acked 1002 Read 8 Done Done Done false
32612023-09-22T23:25:02.379Z 4 Acked 1003 Read 4 Done Done Done false
32622023-09-22T23:25:02.382Z 5 Acked 1004 Write 3 Done Done Done false
32632023-09-22T23:25:02.385Z 6 Acked 1005 Write 5 Done Done Sent false
32642023-09-22T23:25:02.388Z 7 Acked 1006 Read 6 Done Done Sent false
32652023-09-22T23:25:02.391Z 8 Acked 1007 Write 2 Done Done Sent false
32662023-09-22T23:25:02.394Z 9 Acked 1008 Read 7 Done Done Sent false
32672023-09-22T23:25:02.397Z 10 Acked 1009 Write 10 Done Done Sent false
32682023-09-22T23:25:02.400Z 11 Acked 1010 Read 4 Done Done Sent false
32692023-09-22T23:25:02.403Z 12 Acked 1011 Read 6 Done Done Sent false
32702023-09-22T23:25:02.407Z 13 Acked 1012 Write 1 Done Done Sent false
32712023-09-22T23:25:02.409Z 14 Acked 1013 Write 1 Done Done Sent false
32722023-09-22T23:25:02.412Z 15 Acked 1014 Write 7 Done Done Sent false
32732023-09-22T23:25:02.415Z 16 Acked 1015 Write 6 Done Done Sent false
32742023-09-22T23:25:02.418Z 17 Acked 1016 Write 3 Done Done Sent false
32752023-09-22T23:25:02.421Z 18 Acked 1017 Read 1 Done Done Sent false
32762023-09-22T23:25:02.424Z 19 Acked 1018 Flush 0 Done Done Sent false
32772023-09-22T23:25:02.427Z 20 Acked 1019 Write 8 Done Done Sent false
32782023-09-22T23:25:02.430Z 21 Acked 1020 Write 1 Done Done Sent false
32792023-09-22T23:25:02.434Z 22 Acked 1021 Read 4 Done Done Sent false
32802023-09-22T23:25:02.436Z 23 Acked 1022 Write 9 Done Done Sent false
32812023-09-22T23:25:02.440Z 24 Acked 1023 Write 7 Done Done Sent false
32822023-09-22T23:25:02.443Z 25 Acked 1024 Write 3 Done Done Sent false
32832023-09-22T23:25:02.446Z 26 Acked 1025 Flush 0 Done Done Sent false
32842023-09-22T23:25:02.449Z 27 Acked 1026 Write 7 Sent Sent Sent false
32852023-09-22T23:25:02.453Z 28 Acked 1027 Write 6 Sent Sent Sent false
32862023-09-22T23:25:02.456Z 29 Acked 1028 Write 6 Sent Sent Sent false
32872023-09-22T23:25:02.459Z 30 Acked 1029 Write 4 Sent Sent Sent false
32882023-09-22T23:25:02.462Z STATES DS:0 DS:1 DS:2 TOTAL
32892023-09-22T23:25:02.466Z New 0 0 0 0
32902023-09-22T23:25:02.470Z Sent 4 4 25 33
32912023-09-22T23:25:02.473Z Done 26 26 5 57
32922023-09-22T23:25:02.477Z Skipped 0 0 0 0
32932023-09-22T23:25:02.480Z Error 0 0 0 0
32942023-09-22T23:25:02.483ZLast Flush: 1025 1025 0
32952023-09-22T23:25:02.487ZDownstairs last five completed:
32962023-09-22T23:25:02.492ZUpstairs last five completed: 30 29 28 27 26 25
32972023-09-22T23:25:02.495ZWrote out file "/tmp/test_repair_verify.data"
32982023-09-22T23:25:02.498Z
32992023-09-22T23:25:02.501ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
33002023-09-22T23:25:02.504Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:01.112522367Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1253}
33012023-09-22T23:25:02.507Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:01.112818291Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1253}
33022023-09-22T23:25:02.511Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:01.112841909Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1253}
33032023-09-22T23:25:02.514ZError: Difference in extent metadata found!
33042023-09-22T23:25:02.517Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:01.142223028Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1253}
33052023-09-22T23:25:02.520Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:01.142258634Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1253}
33062023-09-22T23:25:02.523Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:01.142278675Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1253}
33072023-09-22T23:25:02.526Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:01.157158198Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1253}
33082023-09-22T23:25:02.530Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:01.157189718Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1253}
33092023-09-22T23:25:02.534Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:01.157209116Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1253}
33102023-09-22T23:25:02.537ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
33112023-09-22T23:25:02.541Z 0 000-019 22 22 22 33 33 33 F F F
33122023-09-22T23:25:02.543Z 1 020-039 24 24 21 34 34 32 F F T <---
33132023-09-22T23:25:02.547Z 2 040-059 24 24 22 34 34 33 F F T <---
33142023-09-22T23:25:02.550Z 3 060-079 22 22 22 33 33 33 F F F
33152023-09-22T23:25:02.552Z 4 080-099 18 18 18 26 26 26 F F F
33162023-09-22T23:25:02.555Z 5 100-119 21 21 21 32 32 32 F F F
33172023-09-22T23:25:02.559Z 6 120-139 24 24 22 35 35 33 T T F <---
33182023-09-22T23:25:02.561Z 7 140-159 22 22 22 33 33 33 F F F
33192023-09-22T23:25:02.564Z 8 160-179 24 24 22 34 34 33 F F T <---
33202023-09-22T23:25:02.567Z 9 180-199 24 24 20 34 34 28 F F F <---
33212023-09-22T23:25:02.570Z 10 200-219 24 24 15 34 34 23 F F F <---
33222023-09-22T23:25:02.574Z 11 220-239 24 24 22 34 34 33 F F F <---
33232023-09-22T23:25:02.577Z 12 240-259 22 22 22 33 33 33 F F F
33242023-09-22T23:25:02.580Z 13 260-279 20 20 20 30 30 30 T T F <---
33252023-09-22T23:25:02.583Z 14 280-299 17 17 17 25 25 25 F F F
33262023-09-22T23:25:02.586Z 15 300-319 16 16 16 24 24 24 F F F
33272023-09-22T23:25:02.590Z 16 320-339 16 16 16 24 24 24 T T F <---
33282023-09-22T23:25:02.596Z 17 340-359 24 24 16 35 35 24 F F F <---
33292023-09-22T23:25:02.599Z 18 360-379 22 22 22 33 33 33 F F F
33302023-09-22T23:25:02.602Z 19 380-399 24 24 22 35 35 33 F F F <---
33312023-09-22T23:25:02.605Z 20 400-419 17 17 17 25 25 25 F F F
33322023-09-22T23:25:02.608Z 21 420-439 20 20 20 29 29 29 F F F
33332023-09-22T23:25:02.611Z 22 440-459 24 24 22 35 35 33 F F F <---
33342023-09-22T23:25:02.614Z 23 460-479 24 24 16 34 34 24 F F F <---
33352023-09-22T23:25:02.617Z 24 480-499 24 24 22 34 34 33 F F T <---
33362023-09-22T23:25:02.620Z 25 500-519 24 24 22 35 35 33 F F F <---
33372023-09-22T23:25:02.623Z 26 520-539 21 21 21 32 32 32 F F F
33382023-09-22T23:25:02.626Z 27 540-559 19 19 19 27 27 27 F F F
33392023-09-22T23:25:02.630Z 28 560-579 22 22 22 33 33 33 F F F
33402023-09-22T23:25:02.633Z 29 580-599 21 21 21 32 32 32 F F F
33412023-09-22T23:25:02.637ZMax gen: 24, Max flush: 35
33422023-09-22T23:25:02.642ZA Difference in extent metadata is expected here
33432023-09-22T23:25:02.645ZOn loop 12
33442023-09-22T23:25:02.649Z
33452023-09-22T23:25:02.652ZVerifying data now
33462023-09-22T23:25:08.907ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
33472023-09-22T23:25:08.910Z434a435
33482023-09-22T23:25:08.914Z> 4,
33492023-09-22T23:25:08.916Z436a438
33502023-09-22T23:25:08.920Z> 2,
33512023-09-22T23:25:08.922Z439,440d440
33522023-09-22T23:25:08.925Z< 1,
33532023-09-22T23:25:08.929Z< 1,
33542023-09-22T23:25:08.931Zdiff found after verify
33552023-09-22T23:25:08.935ZLoop: 12 Downstairs dump after verify (and repair):
33562023-09-22T23:25:08.937Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.832386765Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1261}
33572023-09-22T23:25:08.940Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.832686612Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1261}
33582023-09-22T23:25:08.943Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.832729142Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1261}
33592023-09-22T23:25:08.954Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.851704474Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1261}
33602023-09-22T23:25:08.957Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.851759646Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1261}
33612023-09-22T23:25:08.960Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.851780347Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1261}
33622023-09-22T23:25:08.965Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.862846817Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1261}
33632023-09-22T23:25:08.968Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.862899338Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1261}
33642023-09-22T23:25:08.971Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.862919929Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1261}
33652023-09-22T23:25:08.976ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
33662023-09-22T23:25:08.979Z 0 000-019 22 22 22 33 33 33 F F F
33672023-09-22T23:25:08.982Z 1 020-039 24 24 24 34 34 34 F F F
33682023-09-22T23:25:08.985Z 2 040-059 24 24 24 34 34 34 F F F
33692023-09-22T23:25:08.988Z 3 060-079 22 22 22 33 33 33 F F F
33702023-09-22T23:25:08.991Z 4 080-099 18 18 18 26 26 26 F F F
33712023-09-22T23:25:08.994Z 5 100-119 21 21 21 32 32 32 F F F
33722023-09-22T23:25:08.997Z 6 120-139 25 25 25 36 36 36 F F F
33732023-09-22T23:25:08.999Z 7 140-159 22 22 22 33 33 33 F F F
33742023-09-22T23:25:09.003Z 8 160-179 24 24 24 34 34 34 F F F
33752023-09-22T23:25:09.006Z 9 180-199 24 24 24 34 34 34 F F F
33762023-09-22T23:25:09.008Z 10 200-219 24 24 24 34 34 34 F F F
33772023-09-22T23:25:09.011Z 11 220-239 24 24 24 34 34 34 F F F
33782023-09-22T23:25:09.014Z 12 240-259 22 22 22 33 33 33 F F F
33792023-09-22T23:25:09.017Z 13 260-279 25 25 25 36 36 36 F F F
33802023-09-22T23:25:09.020Z 14 280-299 17 17 17 25 25 25 F F F
33812023-09-22T23:25:09.023Z 15 300-319 16 16 16 24 24 24 F F F
33822023-09-22T23:25:09.026Z 16 320-339 25 25 25 36 36 36 F F F
33832023-09-22T23:25:09.028Z 17 340-359 24 24 24 35 35 35 F F F
33842023-09-22T23:25:09.031Z 18 360-379 22 22 22 33 33 33 F F F
33852023-09-22T23:25:09.034Z 19 380-399 24 24 24 35 35 35 F F F
33862023-09-22T23:25:09.037Z 20 400-419 17 17 17 25 25 25 F F F
33872023-09-22T23:25:09.039Z 21 420-439 20 20 20 29 29 29 F F F
33882023-09-22T23:25:09.042Z 22 440-459 24 24 24 35 35 35 F F F
33892023-09-22T23:25:09.045Z 23 460-479 24 24 24 34 34 34 F F F
33902023-09-22T23:25:09.048Z 24 480-499 24 24 24 34 34 34 F F F
33912023-09-22T23:25:09.051Z 25 500-519 24 24 24 35 35 35 F F F
33922023-09-22T23:25:09.053Z 26 520-539 21 21 21 32 32 32 F F F
33932023-09-22T23:25:09.056Z 27 540-559 19 19 19 27 27 27 F F F
33942023-09-22T23:25:09.059Z 28 560-579 22 22 22 33 33 33 F F F
33952023-09-22T23:25:09.062Z 29 580-599 21 21 21 32 32 32 F F F
33962023-09-22T23:25:09.065ZMax gen: 25, Max flush: 36
33972023-09-22T23:25:09.068Z
33982023-09-22T23:25:09.070ZBegin loop 13
33992023-09-22T23:25:09.073ZDownstairs to restart: 2
34002023-09-22T23:25:09.076Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.912521172Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34012023-09-22T23:25:09.078Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.912927562Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34022023-09-22T23:25:09.081Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.912995907Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34032023-09-22T23:25:09.084Z{"msg":"Crucible stats registered with UUID: 8a2ea902-5d67-4a06-882e-87a2de11fb72","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.913020809Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34042023-09-22T23:25:09.087Z{"msg":"Crucible 8a2ea902-5d67-4a06-882e-87a2de11fb72 has session id: 93f884fd-d141-49b6-83c5-c6a43c259bef","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.913042127Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34052023-09-22T23:25:09.090ZCrucible runtime is spawned
34062023-09-22T23:25:09.095ZThe guest has requested activation with gen:26
34072023-09-22T23:25:09.099Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.913437546Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263,"task":"up_listen"}
34082023-09-22T23:25:09.102Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.913507715Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34092023-09-22T23:25:09.105Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.913552215Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34102023-09-22T23:25:09.108Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.913592967Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263,"looper":"2"}
34112023-09-22T23:25:09.110Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.913827171Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263,"looper":"0"}
34122023-09-22T23:25:09.113Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.914082368Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263,"looper":"1"}
34132023-09-22T23:25:09.116Z{"msg":"[0] 8a2ea902-5d67-4a06-882e-87a2de11fb72 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.914165308Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263,"looper":"0"}
34142023-09-22T23:25:09.119Z{"msg":"[1] 8a2ea902-5d67-4a06-882e-87a2de11fb72 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.914300804Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263,"looper":"1"}
34152023-09-22T23:25:09.122Z{"msg":"8a2ea902-5d67-4a06-882e-87a2de11fb72 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.914615118Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34162023-09-22T23:25:09.124Z{"msg":"Set desired generation to :26","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.914718723Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34172023-09-22T23:25:09.127Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.914771606Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34182023-09-22T23:25:09.130Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.915109924Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34192023-09-22T23:25:09.132Z{"msg":"[0] 8a2ea902-5d67-4a06-882e-87a2de11fb72 (5537aa5f-70cb-46bd-aae2-feaef8418b22) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.915436955Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34202023-09-22T23:25:09.135Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.915511185Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34212023-09-22T23:25:09.139Z{"msg":"[0] client is_active_req TRUE, promote! session 5537aa5f-70cb-46bd-aae2-feaef8418b22","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.915587159Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34222023-09-22T23:25:09.141Z{"msg":"[1] 8a2ea902-5d67-4a06-882e-87a2de11fb72 (5537aa5f-70cb-46bd-aae2-feaef8418b22) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.915673744Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34232023-09-22T23:25:09.144Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.915746886Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34242023-09-22T23:25:09.147Z{"msg":"[1] client is_active_req TRUE, promote! session 5537aa5f-70cb-46bd-aae2-feaef8418b22","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.915844423Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34252023-09-22T23:25:09.150Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.91621282Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34262023-09-22T23:25:09.153Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.916284373Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34272023-09-22T23:25:09.155Z{"msg":"8a2ea902-5d67-4a06-882e-87a2de11fb72 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.916391993Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34282023-09-22T23:25:09.158Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.91645342Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34292023-09-22T23:25:09.161Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.916482125Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34302023-09-22T23:25:09.165Z{"msg":"8a2ea902-5d67-4a06-882e-87a2de11fb72 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.916556589Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34312023-09-22T23:25:09.168Z{"msg":"[0] 8a2ea902-5d67-4a06-882e-87a2de11fb72 (5537aa5f-70cb-46bd-aae2-feaef8418b22) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.918653354Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34322023-09-22T23:25:09.171Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.918722632Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34332023-09-22T23:25:09.173Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:07.918766924Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34342023-09-22T23:25:09.176Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.918806281Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34352023-09-22T23:25:09.180Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.918844334Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34362023-09-22T23:25:09.185Z{"msg":"8a2ea902-5d67-4a06-882e-87a2de11fb72 WaitQuorum WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.918884604Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34372023-09-22T23:25:09.188Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.918923516Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34382023-09-22T23:25:09.192Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.918961792Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34392023-09-22T23:25:09.194Z{"msg":"[1] 8a2ea902-5d67-4a06-882e-87a2de11fb72 (5537aa5f-70cb-46bd-aae2-feaef8418b22) WaitQuorum WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.919126342Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34402023-09-22T23:25:09.197Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.919177139Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34412023-09-22T23:25:09.200Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:07.919217978Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34422023-09-22T23:25:09.203Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.919256024Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34432023-09-22T23:25:09.206Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.919294273Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34442023-09-22T23:25:09.209Z{"msg":"8a2ea902-5d67-4a06-882e-87a2de11fb72 WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.919332556Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34452023-09-22T23:25:09.212Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.919370896Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34462023-09-22T23:25:09.214Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:07.919408916Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34472023-09-22T23:25:10.018Z{"msg":"[2] 8a2ea902-5d67-4a06-882e-87a2de11fb72 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.915293716Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263,"looper":"2"}
34482023-09-22T23:25:10.021Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.915378784Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34492023-09-22T23:25:10.024Z{"msg":"[2] 8a2ea902-5d67-4a06-882e-87a2de11fb72 (5537aa5f-70cb-46bd-aae2-feaef8418b22) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.916422221Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34502023-09-22T23:25:10.027Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.916482331Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34512023-09-22T23:25:10.030Z{"msg":"[2] client is_active_req TRUE, promote! session 5537aa5f-70cb-46bd-aae2-feaef8418b22","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.916533781Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34522023-09-22T23:25:10.033Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.91711525Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34532023-09-22T23:25:10.036Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.917171439Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34542023-09-22T23:25:10.039Z{"msg":"8a2ea902-5d67-4a06-882e-87a2de11fb72 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.917214577Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34552023-09-22T23:25:10.042Z{"msg":"[2] 8a2ea902-5d67-4a06-882e-87a2de11fb72 (5537aa5f-70cb-46bd-aae2-feaef8418b22) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.919586578Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34562023-09-22T23:25:10.044Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.919664161Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34572023-09-22T23:25:10.047Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:08.919730546Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34582023-09-22T23:25:10.050Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.919785011Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34592023-09-22T23:25:10.053Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.919826466Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34602023-09-22T23:25:10.056Z{"msg":"8a2ea902-5d67-4a06-882e-87a2de11fb72 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.919866569Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34612023-09-22T23:25:10.059Z{"msg":"[0]R flush_numbers[0..12]: [33, 34, 34, 33, 26, 32, 36, 33, 34, 34, 34, 34]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.919917754Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34622023-09-22T23:25:10.062Z{"msg":"[0]R generation[0..12]: [22, 24, 24, 22, 18, 21, 25, 22, 24, 24, 24, 24]","v":0,"name":"crucible","level":30,"time":"The guest has finished waiting for activation with:26
34632023-09-22T23:25:10.065Z2023-09-22T23:25:08.919957008Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34642023-09-22T23:25:10.067Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.920067474Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34652023-09-22T23:25:10.070Z{"msg":"[1]R flush_numbers[0..12]: [33, 34, 34, 33, 26, 32, 36, 33, 34, 34, 34, 34]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.920175441Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34662023-09-22T23:25:10.073Z{"msg":"[1]R generation[0..12]: [22, 24, 24, 22, 18, 21, 25, 22, 24, 24, 24, 24]","v":0,"name":"crucibleWait for a query_work_queue command to finish before sending IO
34672023-09-22T23:25:10.076Z","level":30,"time":"2023-09-22T23:25:08.920312708Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34682023-09-22T23:25:10.079Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.920385927Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34692023-09-22T23:25:10.082Z{"msg":"[2]R flush_numbers[0..12]: [33, 34, 34, 33, 26, 32, 36, 33, 34, 34, 34, 34]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.920444069Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34702023-09-22T23:25:10.085Z{"msg":"[2]R generation[0..12]: [22, 24, 24, 22, 18, 21, 25, 22, 24, 24, 24, 24]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.920499266Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34712023-09-22T23:25:10.088Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.920556349Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34722023-09-22T23:25:10.091Z{"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.92061673Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34732023-09-22T23:25:10.093Z{"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.920676686Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34742023-09-22T23:25:10.096Z{"msg":"Next flush: 37","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.920730678Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34752023-09-22T23:25:10.099Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.920789792Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34762023-09-22T23:25:10.102Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.920903308Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34772023-09-22T23:25:10.110Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.92095821Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34782023-09-22T23:25:10.113Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.920995672Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34792023-09-22T23:25:10.116Z{"msg":"8a2ea902-5d67-4a06-882e-87a2de11fb72 is now active with session: 5537aa5f-70cb-46bd-aae2-feaef8418b22","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.921056598Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34802023-09-22T23:25:10.119Z{"msg":"8a2ea902-5d67-4a06-882e-87a2de11fb72 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.92111379Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34812023-09-22T23:25:10.122Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
34822023-09-22T23:25:10.125Z,"time":"2023-09-22T23:25:08.921176403Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34832023-09-22T23:25:10.127Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.921287361Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34842023-09-22T23:25:10.130Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.921344258Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34852023-09-22T23:25:10.133Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.921398248Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34862023-09-22T23:25:10.136Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.921450456Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34872023-09-22T23:25:10.139Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.921506799Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34882023-09-22T23:25:10.142Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.921559377Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34892023-09-22T23:25:10.145Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.921612048Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34902023-09-22T23:25:10.147Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.92166486Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34912023-09-22T23:25:10.150Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:08.921718027Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34922023-09-22T23:25:10.155Z{"msg":"[1] Starts cmd_loop","v":0,"name":"Loading write count information from file "/tmp/test_repair_verify.data"
34932023-09-22T23:25:10.157Zcrucible","level":30Run Repair workload
34942023-09-22T23:25:10.160Z,"time":"2023-09-22T23:25:08.921833535Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1263}
34952023-09-22T23:25:10.163Z01/30 Write block 398 len 12288 data: 3 3 4
34962023-09-22T23:25:10.166Z02/30 Write block 322 len 20480 data: 4 4 4 3 3
34972023-09-22T23:25:10.169Z03/30 Write block 194 len 40960 data: 3 3 2 2 2 2 2 2 4 4
34982023-09-22T23:25:10.171Z04/30 Write block 91 len 20480 data: 4 4 4 4 5
34992023-09-22T23:25:10.174Z05/30 Flush
35002023-09-22T23:25:10.177Z06/30 Flush
35012023-09-22T23:25:10.180Z07/30 Flush
35022023-09-22T23:25:10.183Z08/30 Write block 548 len 16384 data: 2 4 4 4
35032023-09-22T23:25:10.186Z09/30 Read block 335 len 36864
35042023-09-22T23:25:10.189Z10/30 Read block 20 len 28672
35052023-09-22T23:25:10.192Z11/30 Write block 209 len 40960 data: 5 5 6 5 5 4 5 4 4 4
35062023-09-22T23:25:10.195Z12/30 Write block 283 len 20480 data: 3 3 3 4 4
35072023-09-22T23:25:10.198Z13/30 Read block 392 len 40960
35082023-09-22T23:25:10.201Z14/30 Write block 378 len 16384 data: 3 3 4 3
35092023-09-22T23:25:10.204Z15/30 Read block 499 len 8192
35102023-09-22T23:25:10.207Z16/30 Flush
35112023-09-22T23:25:10.210Z17/30 Write block 374 len 24576 data: 2 2 2 3 4 4
35122023-09-22T23:25:10.212Z18/30 Write block 104 len 28672 data: 3 5 5 5 5 5 5
35132023-09-22T23:25:10.216Z19/30 Write block 316 len 32768 data: 6 6 5 5 6 6 5 5
35142023-09-22T23:25:10.219Z20/30 Read block 136 len 8192
35152023-09-22T23:25:10.222Z21/30 Read block 376 len 4096
35162023-09-22T23:25:10.225Z22/30 Read block 172 len 20480
35172023-09-22T23:25:10.228Z23/30 Read block 245 len 36864
35182023-09-22T23:25:10.231Z24/30 Write block 159 len 36864 data: 4 5 5 4 3 3 3 3 4
35192023-09-22T23:25:10.234Z25/30 Read block 184 len 28672
35202023-09-22T23:25:10.237Z26/30 Write block 542 len 28672 data: 4 4 4 3 4 2 3
35212023-09-22T23:25:10.240Z27/30 Read block 222 len 32768
35222023-09-22T23:25:10.242Z28/30 Write block 264 len 16384 data: 3 3 3 4
35232023-09-22T23:25:10.245Z29/30 Write block 595 len 8192 data: 3 3
35242023-09-22T23:25:10.248Z30/30 Read block 111 len 24576
35252023-09-22T23:25:10.251Z----------------------------------------------------------------
35262023-09-22T23:25:10.254Z Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:25
35272023-09-22T23:25:10.257ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
35282023-09-22T23:25:10.260Z 6 Acked 1005 Flush 0 Done Done Sent false
35292023-09-22T23:25:10.263Z 7 Acked 1006 Flush 0 Done Done Sent false
35302023-09-22T23:25:10.266Z 8 Acked 1007 Write 4 Done Done Sent false
35312023-09-22T23:25:10.269Z 9 Acked 1008 Read 9 Done Done Sent false
35322023-09-22T23:25:10.272Z 10 Acked 1009 Read 7 Done Done Sent false
35332023-09-22T23:25:10.275Z 11 Acked 1010 Write 10 Done Done Sent false
35342023-09-22T23:25:10.278Z 12 Acked 1011 Write 5 Done Done Sent false
35352023-09-22T23:25:10.281Z 13 Acked 1012 Read 10 Done Done Sent false
35362023-09-22T23:25:10.284Z 14 Acked 1013 Write 4 Done Done Sent false
35372023-09-22T23:25:10.287Z 15 Acked 1014 Read 2 Done Done Sent false
35382023-09-22T23:25:10.290Z 16 Acked 1015 Flush 0 Done Done Sent false
35392023-09-22T23:25:10.293Z 17 Acked 1016 Write 6 Done Done Sent false
35402023-09-22T23:25:10.317Z 18 Acked 1017 Write 7 Done Done Sent false
35412023-09-22T23:25:10.320Z 19 Acked 1018 Write 8 Done Done Sent false
35422023-09-22T23:25:10.323Z 20 Acked 1019 Read 2 Done Done Sent false
35432023-09-22T23:25:10.326Z 21 Acked 1020 Read 1 Done Done Sent false
35442023-09-22T23:25:10.329Z 22 Acked 1021 Read 5 Done Done Sent false
35452023-09-22T23:25:10.332Z 23 Acked 1022 Read 9 Done Done Sent false
35462023-09-22T23:25:10.335Z 24 Acked 1023 Write 9 Done Done Sent false
35472023-09-22T23:25:10.338Z 25 Acked 1024 Read 7 Done Done Sent false
35482023-09-22T23:25:10.340Z 26 Acked 1025 Write 7 Done Done Sent false
35492023-09-22T23:25:10.343Z 27 Acked 1026 Read 8 Done Done Sent false
35502023-09-22T23:25:10.346Z 28 Acked 1027 Write 4 Done Done Sent false
35512023-09-22T23:25:10.349Z 29 Acked 1028 Write 2 Done Sent Sent false
35522023-09-22T23:25:10.352Z 30 Acked 1029 Read 6 Done Sent Sent false
35532023-09-22T23:25:10.355Z STATES DS:0 DS:1 DS:2 TOTAL
35542023-09-22T23:25:10.358Z New 0 0 0 0
35552023-09-22T23:25:10.360Z Sent 0 2 25 27
35562023-09-22T23:25:10.363Z Done 25 23 0 48
35572023-09-22T23:25:10.366Z Skipped 0 0 0 0
35582023-09-22T23:25:10.369Z Error 0 0 0 0
35592023-09-22T23:25:10.372ZLast Flush: 1015 1015 1004
35602023-09-22T23:25:10.375ZDownstairs last five completed: 1004 1003 1002 1001 1000
35612023-09-22T23:25:10.378ZUpstairs last five completed: 30 29 28 27 26 25
35622023-09-22T23:25:10.381ZWrote out file "/tmp/test_repair_verify.data"
35632023-09-22T23:25:10.383Z
35642023-09-22T23:25:10.386ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
35652023-09-22T23:25:10.389Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:09.066951105Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1264}
35662023-09-22T23:25:10.392Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:09.067220122Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1264}
35672023-09-22T23:25:10.395Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:09.067243088Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1264}
35682023-09-22T23:25:10.398Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:09.099509034Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1264}
35692023-09-22T23:25:10.401Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:09.099564808Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1264}
35702023-09-22T23:25:10.403Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:09.099603867Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1264}
35712023-09-22T23:25:10.406Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:09.122748254Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1264}
35722023-09-22T23:25:10.409Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:09.122818362Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1264}
35732023-09-22T23:25:10.412Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:09.122839764Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1264}
35742023-09-22T23:25:10.415ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
35752023-09-22T23:25:10.418Z 0 000-019 22 22 22 33 33 33 F F F
35762023-09-22T23:25:10.420Z 1 020-039 24 24 24 34 34 34 F F F
35772023-09-22T23:25:10.423Z 2 040-059 24 24 24 34 34 34 F F F
35782023-09-22T23:25:10.426Z 3 060-079 22 22 22 33 33 33 F F F
35792023-09-22T23:25:10.428Z 4 080-099 26 26 26 37 37 37 F F F
35802023-09-22T23:25:10.431Z 5 100-119 21 21 21 32 32 32 T T F <---
35812023-09-22T23:25:10.434Z 6 120-139 25 25 25 36 36 36 F F F
35822023-09-22T23:25:10.436Z 7 140-159 22 22 22 33 33 33 T T F <---
35832023-09-22T23:25:10.439Z 8 160-179 24 24 24 34 34 34 T T F <---
35842023-09-22T23:25:10.442Z 9 180-199 26 26 26 37 37 37 F F F
35852023-09-22T23:25:10.445Z 10 200-219 26 26 26 40 40 37 F F F <---
35862023-09-22T23:25:10.448Z 11 220-239 24 24 24 34 34 34 F F F
35872023-09-22T23:25:10.450Z 12 240-259 22 22 22 33 33 33 F F F
35882023-09-22T23:25:10.453Z 13 260-279 25 25 25 36 36 36 T T F <---
35892023-09-22T23:25:10.456Z 14 280-299 26 26 17 40 40 25 F F F <---
35902023-09-22T23:25:10.459Z 15 300-319 16 16 16 24 24 24 T T F <---
35912023-09-22T23:25:10.465ZError: Difference in extent metadata found!
35922023-09-22T23:25:10.468Z 16 320-339 26 26 26 37 37 37 T T F <---
35932023-09-22T23:25:10.471Z 17 340-359 24 24 24 35 35 35 F F F
35942023-09-22T23:25:10.474Z 18 360-379 26 26 22 40 40 33 T T F <---
35952023-09-22T23:25:10.476Z 19 380-399 26 26 26 40 40 37 F F F <---
35962023-09-22T23:25:10.479Z 20 400-419 26 26 26 37 37 37 F F F
35972023-09-22T23:25:10.482Z 21 420-439 20 20 20 29 29 29 F F F
35982023-09-22T23:25:10.485Z 22 440-459 24 24 24 35 35 35 F F F
35992023-09-22T23:25:10.488Z 23 460-479 24 24 24 34 34 34 F F F
36002023-09-22T23:25:10.491Z 24 480-499 24 24 24 34 34 34 F F F
36012023-09-22T23:25:10.494Z 25 500-519 24 24 24 35 35 35 F F F
36022023-09-22T23:25:10.496Z 26 520-539 21 21 21 32 32 32 F F F
36032023-09-22T23:25:10.499Z 27 540-559 26 26 19 40 40 27 T T F <---
36042023-09-22T23:25:10.502Z 28 560-579 22 22 22 33 33 33 F F F
36052023-09-22T23:25:10.505Z 29 580-599 21 21 21 32 32 32 T T F <---
36062023-09-22T23:25:10.508ZMax gen: 26, Max flush: 40
36072023-09-22T23:25:10.510ZA Difference in extent metadata is expected here
36082023-09-22T23:25:10.513ZOn loop 13
36092023-09-22T23:25:10.516Z
36102023-09-22T23:25:10.519ZVerifying data now
36112023-09-22T23:25:16.601ZNo change after verify
36122023-09-22T23:25:16.604ZLoop: 13 Downstairs dump after verify (and repair):
36132023-09-22T23:25:16.619Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.516253602Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1269}
36142023-09-22T23:25:16.622Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.516580388Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1269}
36152023-09-22T23:25:16.625Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.516605636Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1269}
36162023-09-22T23:25:16.637Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.535011817Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1269}
36172023-09-22T23:25:16.640Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.53506538Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1269}
36182023-09-22T23:25:16.643Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.535086287Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1269}
36192023-09-22T23:25:16.648Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.54584525Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1269}
36202023-09-22T23:25:16.651Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.545897767Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1269}
36212023-09-22T23:25:16.654Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.54591831Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1269}
36222023-09-22T23:25:16.659ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
36232023-09-22T23:25:16.662Z 0 000-019 22 22 22 33 33 33 F F F
36242023-09-22T23:25:16.665Z 1 020-039 24 24 24 34 34 34 F F F
36252023-09-22T23:25:16.667Z 2 040-059 24 24 24 34 34 34 F F F
36262023-09-22T23:25:16.671Z 3 060-079 22 22 22 33 33 33 F F F
36272023-09-22T23:25:16.673Z 4 080-099 26 26 26 37 37 37 F F F
36282023-09-22T23:25:16.676Z 5 100-119 27 27 27 41 41 41 F F F
36292023-09-22T23:25:16.679Z 6 120-139 25 25 25 36 36 36 F F F
36302023-09-22T23:25:16.681Z 7 140-159 27 27 27 41 41 41 F F F
36312023-09-22T23:25:16.684Z 8 160-179 27 27 27 41 41 41 F F F
36322023-09-22T23:25:16.687Z 9 180-199 26 26 26 37 37 37 F F F
36332023-09-22T23:25:16.690Z 10 200-219 26 26 26 40 40 40 F F F
36342023-09-22T23:25:16.693Z 11 220-239 24 24 24 34 34 34 F F F
36352023-09-22T23:25:16.696Z 12 240-259 22 22 22 33 33 33 F F F
36362023-09-22T23:25:16.699Z 13 260-279 27 27 27 41 41 41 F F F
36372023-09-22T23:25:16.701Z 14 280-299 26 26 26 40 40 40 F F F
36382023-09-22T23:25:16.704Z 15 300-319 27 27 27 41 41 41 F F F
36392023-09-22T23:25:16.707Z 16 320-339 27 27 27 41 41 41 F F F
36402023-09-22T23:25:16.710Z 17 340-359 24 24 24 35 35 35 F F F
36412023-09-22T23:25:16.713Z 18 360-379 27 27 27 41 41 41 F F F
36422023-09-22T23:25:16.716Z 19 380-399 26 26 26 40 40 40 F F F
36432023-09-22T23:25:16.719Z 20 400-419 26 26 26 37 37 37 F F F
36442023-09-22T23:25:16.722Z 21 420-439 20 20 20 29 29 29 F F F
36452023-09-22T23:25:16.726Z 22 440-459 24 24 24 35 35 35 F F F
36462023-09-22T23:25:16.729Z 23 460-479 24 24 24 34 34 34 F F F
36472023-09-22T23:25:16.732Z 24 480-499 24 24 24 34 34 34 F F F
36482023-09-22T23:25:16.735Z 25 500-519 24 24 24 35 35 35 F F F
36492023-09-22T23:25:16.738Z 26 520-539 21 21 21 32 32 32 F F F
36502023-09-22T23:25:16.740Z 27 540-559 27 27 27 41 41 41 F F F
36512023-09-22T23:25:16.743Z 28 560-579 22 22 22 33 33 33 F F F
36522023-09-22T23:25:16.746Z 29 580-599 27 27 27 41 41 41 F F F
36532023-09-22T23:25:16.749ZMax gen: 27, Max flush: 41
36542023-09-22T23:25:16.752Z
36552023-09-22T23:25:16.754ZBegin loop 14
36562023-09-22T23:25:16.757ZDownstairs to restart: 0
36572023-09-22T23:25:16.760Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.594660859Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36582023-09-22T23:25:16.763Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.595045466Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36592023-09-22T23:25:16.766Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.595097266Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36602023-09-22T23:25:16.769Z{"msg":"Crucible stats registered with UUID: e9773eab-86b0-429c-bbeb-210ae076d9d9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.595185569Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36612023-09-22T23:25:16.772Z{"msg":"Crucible e9773eab-86b0-429c-bbeb-210ae076d9d9 has session id: ef761a79-3428-4934-8a37-79d7868c41fb","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.595209009Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36622023-09-22T23:25:16.775ZCrucible runtime is spawned
36632023-09-22T23:25:16.778ZThe guest has requested activation with gen:28
36642023-09-22T23:25:16.781Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.595582554Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271,"task":"up_listen"}
36652023-09-22T23:25:16.784Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.595660758Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36662023-09-22T23:25:16.787Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.595713141Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36672023-09-22T23:25:16.790Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.595754533Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271,"looper":"1"}
36682023-09-22T23:25:16.793Z{"msg":"[1] e9773eab-86b0-429c-bbeb-210ae076d9d9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.596190454Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271,"looper":"1"}
36692023-09-22T23:25:16.796Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.596278769Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271,"looper":"2"}
36702023-09-22T23:25:16.799Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.5963699Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271,"looper":"0"}
36712023-09-22T23:25:16.802Z{"msg":"[2] e9773eab-86b0-429c-bbeb-210ae076d9d9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.59642217Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271,"looper":"2"}
36722023-09-22T23:25:16.805Z{"msg":"e9773eab-86b0-429c-bbeb-210ae076d9d9 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.596712145Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36732023-09-22T23:25:16.808Z{"msg":"Set desired generation to :28","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.596763531Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36742023-09-22T23:25:16.811Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.596893464Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36752023-09-22T23:25:16.814Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.597215342Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36762023-09-22T23:25:16.817Z{"msg":"[1] e9773eab-86b0-429c-bbeb-210ae076d9d9 (5bda0ffd-0489-46f3-acba-ffbe5c91f5b5) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.597627858Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36772023-09-22T23:25:16.820Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.597682995Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36782023-09-22T23:25:16.824Z{"msg":"[1] client is_active_req TRUE, promote! session 5bda0ffd-0489-46f3-acba-ffbe5c91f5b5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.597726499Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36792023-09-22T23:25:16.826Z{"msg":"[2] e9773eab-86b0-429c-bbeb-210ae076d9d9 (5bda0ffd-0489-46f3-acba-ffbe5c91f5b5) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.597895327Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36802023-09-22T23:25:16.829Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.597946011Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36812023-09-22T23:25:16.833Z{"msg":"[2] client is_active_req TRUE, promote! session 5bda0ffd-0489-46f3-acba-ffbe5c91f5b5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.597966082Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36822023-09-22T23:25:16.836Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.59842401Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36832023-09-22T23:25:16.839Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.598487046Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36842023-09-22T23:25:16.842Z{"msg":"e9773eab-86b0-429c-bbeb-210ae076d9d9 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.59861356Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36852023-09-22T23:25:16.845Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.598750866Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36862023-09-22T23:25:16.848Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.598812893Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36872023-09-22T23:25:16.851Z{"msg":"e9773eab-86b0-429c-bbeb-210ae076d9d9 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.598839223Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36882023-09-22T23:25:16.854Z{"msg":"[1] e9773eab-86b0-429c-bbeb-210ae076d9d9 (5bda0ffd-0489-46f3-acba-ffbe5c91f5b5) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.600740842Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36892023-09-22T23:25:16.857Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.600798365Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36902023-09-22T23:25:16.860Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:15.600822053Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36912023-09-22T23:25:16.864Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.60088282Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36922023-09-22T23:25:16.867Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.600944167Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36932023-09-22T23:25:16.870Z{"msg":"e9773eab-86b0-429c-bbeb-210ae076d9d9 New WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.600985775Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36942023-09-22T23:25:16.873Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.601027065Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36952023-09-22T23:25:16.876Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.60106518Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36962023-09-22T23:25:16.879Z{"msg":"[2] e9773eab-86b0-429c-bbeb-210ae076d9d9 (5bda0ffd-0489-46f3-acba-ffbe5c91f5b5) New WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.601237648Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36972023-09-22T23:25:16.882Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.601288861Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36982023-09-22T23:25:16.885Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:15.60132953Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
36992023-09-22T23:25:16.888Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.601368214Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37002023-09-22T23:25:16.891Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.601407076Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37012023-09-22T23:25:16.894Z{"msg":"e9773eab-86b0-429c-bbeb-210ae076d9d9 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.601445663Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37022023-09-22T23:25:16.897Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.601488183Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37032023-09-22T23:25:16.901Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:15.601526162Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37042023-09-22T23:25:17.702Z{"msg":"[0] e9773eab-86b0-429c-bbeb-210ae076d9d9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.598550854Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271,"looper":"0"}
37052023-09-22T23:25:17.705Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.59865058Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37062023-09-22T23:25:17.708Z{"msg":"[0] e9773eab-86b0-429c-bbeb-210ae076d9d9 (5bda0ffd-0489-46f3-acba-ffbe5c91f5b5) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.5996336Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37072023-09-22T23:25:17.711Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.599696458Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37082023-09-22T23:25:17.714Z{"msg":"[0] client is_active_req TRUE, promote! session 5bda0ffd-0489-46f3-acba-ffbe5c91f5b5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.599721961Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37092023-09-22T23:25:17.717Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.6003499Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37102023-09-22T23:25:17.720Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.600410951Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37112023-09-22T23:25:17.723Z{"msg":"e9773eab-86b0-429c-bbeb-210ae076d9d9 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.60045494Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37122023-09-22T23:25:17.726Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.602631192Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37132023-09-22T23:25:17.729Z{"msg":"[0] e9773eab-86b0-429c-bbeb-210ae076d9d9 (5bda0ffd-0489-46f3-acba-ffbe5c91f5b5) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.602866702Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37142023-09-22T23:25:17.732Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.602926891Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37152023-09-22T23:25:17.735Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:16.602971387Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37162023-09-22T23:25:17.738Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.60299276Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37172023-09-22T23:25:17.741Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.603031054Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37182023-09-22T23:25:17.744Z{"msg":"e9773eab-86b0-429c-bbeb-210ae076d9d9 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.603071619Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37192023-09-22T23:25:17.748Z{"msg":"[0]R flush_numbers[0..12]: [33, 34, 34, 33, 37, 41, 36, 41, 41, 37, 40, 34]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.60311233Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37202023-09-22T23:25:17.751Z{"msg":"[0]R generation[0..12]: [22, 24, 24, 22, 26, 27, 25, 27, 27, 26, 26, 24]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.603152735Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37212023-09-22T23:25:17.753Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.603193666Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37222023-09-22T23:25:17.757Z{"msg":"[1]R flush_numbers[0..12]: [33, 34, 34, 33, 37, 41, 36, 41, 41, 37, 40, 34]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.603238838Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37232023-09-22T23:25:17.759Z{"msg":"[1]R generation[0..12]: [22, 24, 24, 22, 26, 27, 25, 27, 27, 26, 26, 24]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.603281034Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37242023-09-22T23:25:17.762Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.603321928Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37252023-09-22T23:25:17.765Z{"msg":"[2]R flush_numbers[0..12]: [33, 34, 34, 33, 37, 41, 36, 41, 41, 37, 40, 34]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:28
37262023-09-22T23:25:17.768ZWait for a query_work_queue command to finish before sending IO
37272023-09-22T23:25:17.771Z,"time":"2023-09-22T23:25:16.603394612Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37282023-09-22T23:25:17.774Z{"msg":"[2]R generation[0..12]: [22, 24, 24, 22, 26, 27, 25, 27, 27, 26, 26, 24]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.603586351Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37292023-09-22T23:25:17.777Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.603662903Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37302023-09-22T23:25:17.780Z{"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.603732256Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37312023-09-22T23:25:17.782Z{"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.603787375Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37322023-09-22T23:25:17.785Z{"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.603807887Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37332023-09-22T23:25:17.788Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.603845717Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37342023-09-22T23:25:17.791Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.603898428Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37352023-09-22T23:25:17.794Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.603957022Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37362023-09-22T23:25:17.797Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.603978789Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37372023-09-22T23:25:17.801Z{"msg":"e9773eab-86b0-429c-bbeb-210ae076d9d9 is now active with session: 5bda0ffd-0489-46f3-acba-ffbe5c91f5b5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.603998398Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37382023-09-22T23:25:17.804Z{"msg":"e9773eab-86b0-429c-bbeb-210ae076d9d9 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.604018431Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37392023-09-22T23:25:17.807Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.604056699Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37402023-09-22T23:25:17.810Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.60409585Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37412023-09-22T23:25:17.813Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.604142111Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37422023-09-22T23:25:17.816Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.604180894Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37432023-09-22T23:25:17.819Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.604228447Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37442023-09-22T23:25:17.822Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.604277578Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37452023-09-22T23:25:17.825Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.60431637Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37462023-09-22T23:25:17.829Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.604354939Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37472023-09-22T23:25:17.832Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.604375646Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37482023-09-22T23:25:17.835Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.60439315Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37492023-09-22T23:25:17.838Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.604435774Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1271}
37502023-09-22T23:25:17.841ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
37512023-09-22T23:25:17.844ZLoading write count information from file "/tmp/test_repair_verify.data"
37522023-09-22T23:25:17.847ZRun Repair workload
37532023-09-22T23:25:17.850Z01/30 Write block 96 len 12288 data: 5 4 4
37542023-09-22T23:25:17.853Z02/30 Read block 510 len 32768
37552023-09-22T23:25:17.856Z03/30 Read block 69 len 32768
37562023-09-22T23:25:17.859Z04/30 Write block 261 len 28672 data: 3 4 4 4 4 4 5
37572023-09-22T23:25:17.862Z05/30 Write block 395 len 32768 data: 3 3 3 4 4 5 4 5
37582023-09-22T23:25:17.865Z06/30 Read block 206 len 20480
37592023-09-22T23:25:17.868Z07/30 Write block 95 len 8192 data: 6 6
37602023-09-22T23:25:17.871Z08/30 Read block 363 len 28672
37612023-09-22T23:25:17.874Z09/30 Write block 282 len 24576 data: 3 4 4 4 5 5
37622023-09-22T23:25:17.877Z10/30 Write block 432 len 28672 data: 4 3 2 2 2 2 2
37632023-09-22T23:25:17.880Z11/30 Read block 181 len 20480
37642023-09-22T23:25:17.882Z12/30 Write block 477 len 24576 data: 2 2 2 2 3 3
37652023-09-22T23:25:17.886Z13/30 Write block 592 len 20480 data: 2 3 3 4 4
37662023-09-22T23:25:17.888Z14/30 Read block 520 len 36864
37672023-09-22T23:25:17.892Z15/30 Write block 513 len 28672 data: 4 5 5 4 4 4 3
37682023-09-22T23:25:17.895Z16/30 Flush
37692023-09-22T23:25:17.897Z17/30 Write block 45 len 28672 data: 5 4 5 5 5 4 4
37702023-09-22T23:25:17.900Z18/30 Read block 124 len 8192
37712023-09-22T23:25:17.903Z19/30 Read block 202 len 36864
37722023-09-22T23:25:17.906Z20/30 Read block 372 len 32768
37732023-09-22T23:25:17.909Z21/30 Write block 432 len 28672 data: 5 4 3 3 3 3 3
37742023-09-22T23:25:17.912Z22/30 Read block 96 len 32768
37752023-09-22T23:25:17.915Z23/30 Write block 13 len 12288 data: 3 4 4
37762023-09-22T23:25:17.918Z24/30 Read block 177 len 16384
37772023-09-22T23:25:17.921Z25/30 Write block 351 len 32768 data: 3 3 3 3 4 3 3 3
37782023-09-22T23:25:17.924Z26/30 Write block 44 len 8192 data: 6 6
37792023-09-22T23:25:17.927Z27/30 Write block 491 len 8192 data: 5 4
37802023-09-22T23:25:17.930Z28/30 Write block 443 len 36864 data: 3 5 5 5 3 4 4 4 3
37812023-09-22T23:25:17.933Z29/30 Read block 80 len 32768
37822023-09-22T23:25:17.936Z30/30 Write block 57 len 16384 data: 3 3 3 4
37832023-09-22T23:25:17.939Z----------------------------------------------------------------
37842023-09-22T23:25:17.942Z Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30
37852023-09-22T23:25:17.945ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
37862023-09-22T23:25:17.947Z 1 Acked 1000 Write 3 Done Done Done false
37872023-09-22T23:25:17.950Z 2 Acked 1001 Read 8 Done Done Done false
37882023-09-22T23:25:17.953Z 3 Acked 1002 Read 8 Done Done Done false
37892023-09-22T23:25:17.957Z 4 Acked 1003 Write 7 Done Done Done false
37902023-09-22T23:25:17.960Z 5 Acked 1004 Write 8 Sent Done Done false
37912023-09-22T23:25:17.963Z 6 Acked 1005 Read 5 Sent Done Done false
37922023-09-22T23:25:17.966Z 7 Acked 1006 Write 2 Sent Done Done false
37932023-09-22T23:25:17.969Z 8 Acked 1007 Read 7 Sent Done Done false
37942023-09-22T23:25:17.972Z 9 Acked 1008 Write 6 Sent Done Done false
37952023-09-22T23:25:17.975Z 10 Acked 1009 Write 7 Sent Done Done false
37962023-09-22T23:25:17.978Z 11 Acked 1010 Read 5 Sent Done Done false
37972023-09-22T23:25:17.981Z 12 Acked 1011 Write 6 Sent Done Done false
37982023-09-22T23:25:17.985Z 13 Acked 1012 Write 5 Sent Done Done false
37992023-09-22T23:25:17.988Z 14 Acked 1013 Read 9 Sent Done Done false
38002023-09-22T23:25:17.991Z 15 Acked 1014 Write 7 Sent Done Done false
38012023-09-22T23:25:17.994Z 16 Acked 1015 Flush 0 Sent Done Done false
38022023-09-22T23:25:17.997Z 17 Acked 1016 Write 7 Sent Done Done false
38032023-09-22T23:25:18.000Z 18 Acked 1017 Read 2 Sent Done Done false
38042023-09-22T23:25:18.005Z 19 Acked 1018 Read 9 Sent Done Done false
38052023-09-22T23:25:18.008Z 20 Acked 1019 Read 8 Sent Done Done false
38062023-09-22T23:25:18.011Z 21 Acked 1020 Write 7 Sent Done Done false
38072023-09-22T23:25:18.014Z 22 Acked 1021 Read 8 Sent Done Done false
38082023-09-22T23:25:18.018Z 23 Acked 1022 Write 3 Sent Done Done false
38092023-09-22T23:25:18.021Z 24 Acked 1023 Read 4 Sent Done Done false
38102023-09-22T23:25:18.025Z 25 Acked 1024 Write 8 Sent Done Done false
38112023-09-22T23:25:18.028Z 26 Acked 1025 Write 2 Sent Done Done false
38122023-09-22T23:25:18.031Z 27 Acked 1026 Write 2 Sent Done Done false
38132023-09-22T23:25:18.034Z 28 Acked 1027 Write 9 Sent Done Done false
38142023-09-22T23:25:18.037Z 29 Acked 1028 Read 8 Sent Done Done false
38152023-09-22T23:25:18.040Z 30 Acked 1029 Write 4 Sent Sent Sent false
38162023-09-22T23:25:18.043Z STATES DS:0 DS:1 DS:2 TOTAL
38172023-09-22T23:25:18.046Z New 0 0 0 0
38182023-09-22T23:25:18.049Z Sent 26 1 1 28
38192023-09-22T23:25:18.052Z Done 4 29 29 62
38202023-09-22T23:25:18.055Z Skipped 0 0 0 0
38212023-09-22T23:25:18.059Z Error 0 0 0 0
38222023-09-22T23:25:18.062ZLast Flush: 0 1015 1015
38232023-09-22T23:25:18.065ZDownstairs last five completed:
38242023-09-22T23:25:18.070ZUpstairs last five completed: 30 29 28 27 26 25
38252023-09-22T23:25:18.074ZWrote out file "/tmp/test_repair_verify.data"
38262023-09-22T23:25:18.077Z
38272023-09-22T23:25:18.080ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
38282023-09-22T23:25:18.083Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.746747878Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1272}
38292023-09-22T23:25:18.086Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.747091474Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1272}
38302023-09-22T23:25:18.089Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.747114706Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1272}
38312023-09-22T23:25:18.091Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.783332147Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1272}
38322023-09-22T23:25:18.094Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.783365156Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1272}
38332023-09-22T23:25:18.097Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.783477282Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1272}
38342023-09-22T23:25:18.101Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.805675378Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1272}
38352023-09-22T23:25:18.104Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.80573255Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1272}
38362023-09-22T23:25:18.107Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:16.805772452Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1272}
38372023-09-22T23:25:18.110ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38382023-09-22T23:25:18.113Z 0 000-019 22 22 22 33 33 33 F T T <---
38392023-09-22T23:25:18.117Z 1 020-039 24 24 24 34 34 34 F F F
38402023-09-22T23:25:18.120Z 2 040-059 24 24 24 34 34 34 F T T <---
38412023-09-22T23:25:18.123Z 3 060-079 22 22 22 33 33 33 F T T <---
38422023-09-22T23:25:18.126Z 4 080-099 26 28 28 37 42 42 T F F <---
38432023-09-22T23:25:18.129Z 5 100-119 27 27 27 41 41 41 F F F
38442023-09-22T23:25:18.132Z 6 120-139 25 25 25 36 36 36 F F F
38452023-09-22T23:25:18.135ZError: Difference in extent metadata found!
38462023-09-22T23:25:18.138Z 7 140-159 27 27 27 41 41 41 F F F
38472023-09-22T23:25:18.141Z 8 160-179 27 27 27 41 41 41 F F F
38482023-09-22T23:25:18.144Z 9 180-199 26 26 26 37 37 37 F F F
38492023-09-22T23:25:18.147Z 10 200-219 26 26 26 40 40 40 F F F
38502023-09-22T23:25:18.150Z 11 220-239 24 24 24 34 34 34 F F F
38512023-09-22T23:25:18.153Z 12 240-259 22 22 22 33 33 33 F F F
38522023-09-22T23:25:18.156Z 13 260-279 27 28 28 41 42 42 T F F <---
38532023-09-22T23:25:18.159Z 14 280-299 26 28 28 40 42 42 F F F <---
38542023-09-22T23:25:18.161Z 15 300-319 27 27 27 41 41 41 F F F
38552023-09-22T23:25:18.164Z 16 320-339 27 27 27 41 41 41 F F F
38562023-09-22T23:25:18.167Z 17 340-359 24 24 24 35 35 35 F T T <---
38572023-09-22T23:25:18.170Z 18 360-379 27 27 27 41 41 41 F F F
38582023-09-22T23:25:18.173Z 19 380-399 26 28 28 40 42 42 F F F <---
38592023-09-22T23:25:18.176Z 20 400-419 26 28 28 37 42 42 F F F <---
38602023-09-22T23:25:18.178Z 21 420-439 20 28 28 29 42 42 F T T <---
38612023-09-22T23:25:18.181Z 22 440-459 24 24 24 35 35 35 F T T <---
38622023-09-22T23:25:18.184Z 23 460-479 24 28 28 34 42 42 F F F <---
38632023-09-22T23:25:18.187Z 24 480-499 24 28 28 34 42 42 F T T <---
38642023-09-22T23:25:18.190Z 25 500-519 24 28 28 35 42 42 F F F <---
38652023-09-22T23:25:18.193Z 26 520-539 21 21 21 32 32 32 F F F
38662023-09-22T23:25:18.196Z 27 540-559 27 27 27 41 41 41 F F F
38672023-09-22T23:25:18.199Z 28 560-579 22 22 22 33 33 33 F F F
38682023-09-22T23:25:18.202Z 29 580-599 27 28 28 41 42 42 F F F <---
38692023-09-22T23:25:18.204ZMax gen: 28, Max flush: 42
38702023-09-22T23:25:18.207ZA Difference in extent metadata is expected here
38712023-09-22T23:25:18.210ZOn loop 14
38722023-09-22T23:25:18.213Z
38732023-09-22T23:25:18.216ZVerifying data now
38742023-09-22T23:25:24.517ZNo change after verify
38752023-09-22T23:25:24.520ZLoop: 14 Downstairs dump after verify (and repair):
38762023-09-22T23:25:24.536Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.433175801Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1277}
38772023-09-22T23:25:24.538Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.43348164Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1277}
38782023-09-22T23:25:24.541Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.433506955Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1277}
38792023-09-22T23:25:24.555Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.452582232Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1277}
38802023-09-22T23:25:24.558Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.452637624Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1277}
38812023-09-22T23:25:24.561Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.452658025Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1277}
38822023-09-22T23:25:24.566Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.463712851Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1277}
38832023-09-22T23:25:24.569Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.463766927Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1277}
38842023-09-22T23:25:24.571Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.463787135Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1277}
38852023-09-22T23:25:24.577ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
38862023-09-22T23:25:24.580Z 0 000-019 29 29 29 43 43 43 F F F
38872023-09-22T23:25:24.583Z 1 020-039 24 24 24 34 34 34 F F F
38882023-09-22T23:25:24.586Z 2 040-059 29 29 29 43 43 43 F F F
38892023-09-22T23:25:24.589Z 3 060-079 29 29 29 43 43 43 F F F
38902023-09-22T23:25:24.593Z 4 080-099 28 28 28 42 42 42 F F F
38912023-09-22T23:25:24.596Z 5 100-119 27 27 27 41 41 41 F F F
38922023-09-22T23:25:24.622Z 6 120-139 25 25 25 36 36 36 F F F
38932023-09-22T23:25:24.625Z 7 140-159 27 27 27 41 41 41 F F F
38942023-09-22T23:25:24.628Z 8 160-179 27 27 27 41 41 41 F F F
38952023-09-22T23:25:24.631Z 9 180-199 26 26 26 37 37 37 F F F
38962023-09-22T23:25:24.634Z 10 200-219 26 26 26 40 40 40 F F F
38972023-09-22T23:25:24.637Z 11 220-239 24 24 24 34 34 34 F F F
38982023-09-22T23:25:24.639Z 12 240-259 22 22 22 33 33 33 F F F
38992023-09-22T23:25:24.642Z 13 260-279 28 28 28 42 42 42 F F F
39002023-09-22T23:25:24.645Z 14 280-299 28 28 28 42 42 42 F F F
39012023-09-22T23:25:24.647Z 15 300-319 27 27 27 41 41 41 F F F
39022023-09-22T23:25:24.650Z 16 320-339 27 27 27 41 41 41 F F F
39032023-09-22T23:25:24.653Z 17 340-359 29 29 29 43 43 43 F F F
39042023-09-22T23:25:24.656Z 18 360-379 27 27 27 41 41 41 F F F
39052023-09-22T23:25:24.658Z 19 380-399 28 28 28 42 42 42 F F F
39062023-09-22T23:25:24.661Z 20 400-419 28 28 28 42 42 42 F F F
39072023-09-22T23:25:24.668Z 21 420-439 29 29 29 43 43 43 F F F
39082023-09-22T23:25:24.670Z 22 440-459 29 29 29 43 43 43 F F F
39092023-09-22T23:25:24.673Z 23 460-479 28 28 28 42 42 42 F F F
39102023-09-22T23:25:24.676Z 24 480-499 29 29 29 43 43 43 F F F
39112023-09-22T23:25:24.678Z 25 500-519 28 28 28 42 42 42 F F F
39122023-09-22T23:25:24.681Z 26 520-539 21 21 21 32 32 32 F F F
39132023-09-22T23:25:24.684Z 27 540-559 27 27 27 41 41 41 F F F
39142023-09-22T23:25:24.686Z 28 560-579 22 22 22 33 33 33 F F F
39152023-09-22T23:25:24.689Z 29 580-599 28 28 28 42 42 42 F F F
39162023-09-22T23:25:24.692ZMax gen: 29, Max flush: 43
39172023-09-22T23:25:24.694Z
39182023-09-22T23:25:24.697ZBegin loop 15
39192023-09-22T23:25:24.700ZDownstairs to restart: 1
39202023-09-22T23:25:24.702Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.515198086Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39212023-09-22T23:25:24.705Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.515575432Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39222023-09-22T23:25:24.708Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.515629324Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39232023-09-22T23:25:24.711Z{"msg":"Crucible stats registered with UUID: ba7b9c9a-d2d9-453e-8670-94b9ce87a55b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.515694589Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39242023-09-22T23:25:24.714Z{"msg":"Crucible ba7b9c9a-d2d9-453e-8670-94b9ce87a55b has session id: f9dc65fe-4198-411e-9687-d5dd402c9097","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.515768315Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39252023-09-22T23:25:24.717ZCrucible runtime is spawned
39262023-09-22T23:25:24.720Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.51595333Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279,"task":"up_listen"}
39272023-09-22T23:25:24.723Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":The guest has requested activation with gen:30
39282023-09-22T23:25:24.725Z30,"time":"2023-09-22T23:25:23.516106741Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39292023-09-22T23:25:24.728Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.516160457Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39302023-09-22T23:25:24.730Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.516202235Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279,"looper":"1"}
39312023-09-22T23:25:24.733Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.516404387Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279,"looper":"2"}
39322023-09-22T23:25:24.736Z{"msg":"[2] ba7b9c9a-d2d9-453e-8670-94b9ce87a55b looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.516595062Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279,"looper":"2"}
39332023-09-22T23:25:24.739Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.51667016Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279,"looper":"0"}
39342023-09-22T23:25:24.742Z{"msg":"[0] ba7b9c9a-d2d9-453e-8670-94b9ce87a55b looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.516873079Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279,"looper":"0"}
39352023-09-22T23:25:24.745Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.516925619Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39362023-09-22T23:25:24.748Z{"msg":"ba7b9c9a-d2d9-453e-8670-94b9ce87a55b active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.517090985Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39372023-09-22T23:25:24.751Z{"msg":"Set desired generation to :30","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.517141281Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39382023-09-22T23:25:24.754Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.517462744Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39392023-09-22T23:25:24.761Z{"msg":"[2] ba7b9c9a-d2d9-453e-8670-94b9ce87a55b (a940415d-0c92-48e0-a6d5-c788656afd83) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.517750003Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39402023-09-22T23:25:24.764Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.517817506Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39412023-09-22T23:25:24.767Z{"msg":"[2] client is_active_req TRUE, promote! session a940415d-0c92-48e0-a6d5-c788656afd83","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.51788674Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39422023-09-22T23:25:24.770Z{"msg":"[0] ba7b9c9a-d2d9-453e-8670-94b9ce87a55b (a940415d-0c92-48e0-a6d5-c788656afd83) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.517960858Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39432023-09-22T23:25:24.773Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.518051584Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39442023-09-22T23:25:24.775Z{"msg":"[0] client is_active_req TRUE, promote! session a940415d-0c92-48e0-a6d5-c788656afd83","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.518108485Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39452023-09-22T23:25:24.778Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.518503943Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39462023-09-22T23:25:24.782Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.518607016Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39472023-09-22T23:25:24.785Z{"msg":"ba7b9c9a-d2d9-453e-8670-94b9ce87a55b WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.518693896Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39482023-09-22T23:25:24.788Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.51882603Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39492023-09-22T23:25:24.791Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.518867373Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39502023-09-22T23:25:24.795Z{"msg":"ba7b9c9a-d2d9-453e-8670-94b9ce87a55b WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.518952953Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39512023-09-22T23:25:24.798Z{"msg":"[0] ba7b9c9a-d2d9-453e-8670-94b9ce87a55b (a940415d-0c92-48e0-a6d5-c788656afd83) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.520754936Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39522023-09-22T23:25:24.800Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.520816066Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39532023-09-22T23:25:24.803Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:23.520872858Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39542023-09-22T23:25:24.806Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.52092627Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39552023-09-22T23:25:24.809Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.520989612Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39562023-09-22T23:25:24.812Z{"msg":"ba7b9c9a-d2d9-453e-8670-94b9ce87a55b WaitQuorum New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.521052073Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39572023-09-22T23:25:24.814Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.521088716Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39582023-09-22T23:25:24.817Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.521116683Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39592023-09-22T23:25:24.820Z{"msg":"[2] ba7b9c9a-d2d9-453e-8670-94b9ce87a55b (a940415d-0c92-48e0-a6d5-c788656afd83) WaitQuorum New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.521225786Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39602023-09-22T23:25:24.823Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.521253025Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39612023-09-22T23:25:24.825Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:23.521271513Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39622023-09-22T23:25:24.829Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.52128899Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39632023-09-22T23:25:24.833Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.521307342Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39642023-09-22T23:25:24.836Z{"msg":"ba7b9c9a-d2d9-453e-8670-94b9ce87a55b WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.52132547Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39652023-09-22T23:25:24.838Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.521343458Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39662023-09-22T23:25:24.841Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:23.521360595Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39672023-09-22T23:25:25.620Z{"msg":"[1] ba7b9c9a-d2d9-453e-8670-94b9ce87a55b looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.51744041Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279,"looper":"1"}
39682023-09-22T23:25:25.644Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.517522403Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39692023-09-22T23:25:25.647Z{"msg":"[1] ba7b9c9a-d2d9-453e-8670-94b9ce87a55b (a940415d-0c92-48e0-a6d5-c788656afd83) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.51872169Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39702023-09-22T23:25:25.650Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.518776967Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39712023-09-22T23:25:25.653Z{"msg":"[1] client is_active_req TRUE, promote! session a940415d-0c92-48e0-a6d5-c788656afd83","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.51882076Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39722023-09-22T23:25:25.656Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.519433352Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39732023-09-22T23:25:25.659Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.519576045Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39742023-09-22T23:25:25.662Z{"msg":"ba7b9c9a-d2d9-453e-8670-94b9ce87a55b WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.519624261Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39752023-09-22T23:25:25.665Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.521836702Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39762023-09-22T23:25:25.668Z{"msg":"[1] ba7b9c9a-d2d9-453e-8670-94b9ce87a55b (a940415d-0c92-48e0-a6d5-c788656afd83) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.521974642Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39772023-09-22T23:25:25.670Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.522027536Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39782023-09-22T23:25:25.673Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:24.522049923Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39792023-09-22T23:25:25.676Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.522089445Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39802023-09-22T23:25:25.679Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.522110759Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39812023-09-22T23:25:25.682Z{"msg":"ba7b9c9a-d2d9-453e-8670-94b9ce87a55b WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.522149551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39822023-09-22T23:25:25.685Z{"msg":"[0]R flush_numbers[0..12]: [43, 34, 43, 43, 42, 41, 36, 41, 41, 37, 40, 34]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.522189721Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39832023-09-22T23:25:25.690Z{"msg":"[0]R generation[0..12]: [29, 24, 29, 29, 28, 27, 25, 27, 27, 26, 26, 24]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.522230212Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39842023-09-22T23:25:25.693Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.522271609Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39852023-09-22T23:25:25.696Z{"msg":"[1]R flush_numbers[0..12]: [43, 34, 43, 43, 42, 41, 36, 41, 41, 37, 40, 34]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.522316191Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39862023-09-22T23:25:25.699Z{"msg":"[1]R generation[0..12]: [29, 24, 29, 29, 28, 27, 25, 27, 27, 26, 26, 24]","v":0,"The guest has finished waiting for activation with:30
39872023-09-22T23:25:25.702Zname":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
39882023-09-22T23:25:25.705Z,"time":"2023-09-22T23:25:24.522361022Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39892023-09-22T23:25:25.708Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.522428662Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39902023-09-22T23:25:25.711Z{"msg":"[2]R flush_numbers[0..12]: [43, 34, 43, 43, 42, 41, 36, 41, 41, 37, 40, 34]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.522508074Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39912023-09-22T23:25:25.713Z{"msg":"[2]R generation[0..12]: [29, 24, 29, 29, 28, 27, 25, 27, 27, 26, 26, 24]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.52258168Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39922023-09-22T23:25:25.716Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.522644925Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39932023-09-22T23:25:25.719Z{"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.522687767Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39942023-09-22T23:25:25.722Z{"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.522737299Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39952023-09-22T23:25:25.725Z{"msg":"Next flush: 44","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.522757663Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39962023-09-22T23:25:25.728Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.52277509Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39972023-09-22T23:25:25.730Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.522794019Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39982023-09-22T23:25:25.733Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.522832254Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
39992023-09-22T23:25:25.736Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.522852321Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
40002023-09-22T23:25:25.739Z{"msg":"ba7b9c9a-d2d9-453e-8670-94b9ce87a55b is now active with session: a940415d-0c92-48e0-a6d5-c788656afd83","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.522900375Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
40012023-09-22T23:25:25.742Z{"msg":"ba7b9c9a-d2d9-453e-8670-94b9ce87a55b Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.522946997Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
40022023-09-22T23:25:25.744Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.522991483Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
40032023-09-22T23:25:25.747Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.523032524Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
40042023-09-22T23:25:25.750Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.523052383Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
40052023-09-22T23:25:25.753Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.523093558Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
40062023-09-22T23:25:25.755Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.523213596Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
40072023-09-22T23:25:25.758Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.523261848Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
40082023-09-22T23:25:25.761Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.523310846Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
40092023-09-22T23:25:25.764Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
40102023-09-22T23:25:25.767Z,"time":"2023-09-22T23:25:24.523330596Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
40112023-09-22T23:25:25.770Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.523349588Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
40122023-09-22T23:25:25.773Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.523385912Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
40132023-09-22T23:25:25.776Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.523424655Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1279}
40142023-09-22T23:25:25.779ZLoading write count information from file "/tmp/test_repair_verify.data"
40152023-09-22T23:25:25.782ZRun Repair workload
40162023-09-22T23:25:25.785Z01/30 Write block 588 len 24576 data: 2 2 2 2 3 4
40172023-09-22T23:25:25.787Z02/30 Read block 538 len 8192
40182023-09-22T23:25:25.791Z03/30 Write block 137 len 28672 data: 4 3 3 4 4 4 4
40192023-09-22T23:25:25.794Z04/30 Read block 276 len 4096
40202023-09-22T23:25:25.797Z05/30 Write block 506 len 36864 data: 4 3 2 4 3 3 4 5 6
40212023-09-22T23:25:25.800Z06/30 Write block 227 len 16384 data: 3 3 3 5
40222023-09-22T23:25:25.802Z07/30 Read block 211 len 12288
40232023-09-22T23:25:25.806Z08/30 Write block 190 len 32768 data: 5 4 3 3 4 4 3 3
40242023-09-22T23:25:25.809Z09/30 Write block 276 len 12288 data: 6 3 4
40252023-09-22T23:25:25.811Z10/30 Read block 584 len 16384
40262023-09-22T23:25:25.814Z11/30 Write block 463 len 12288 data: 3 2 3
40272023-09-22T23:25:25.817Z12/30 Write block 208 len 24576 data: 6 6 6 7 6 6
40282023-09-22T23:25:25.820Z13/30 Read block 483 len 32768
40292023-09-22T23:25:25.822Z14/30 Write block 252 len 40960 data: 4 3 3 2 2 2 2 2 2 4
40302023-09-22T23:25:25.825Z15/30 Read block 136 len 16384
40312023-09-22T23:25:25.828Z16/30 Write block 475 len 20480 data: 2 2 3 3 3
40322023-09-22T23:25:25.831Z17/30 Read block 265 len 24576
40332023-09-22T23:25:25.834Z18/30 Read block 397 len 16384
40342023-09-22T23:25:25.837Z19/30 Flush
40352023-09-22T23:25:25.840Z20/30 Write block 170 len 36864 data: 4 4 3 3 3 3 3 2 2
40362023-09-22T23:25:25.843Z21/30 Write block 226 len 20480 data: 4 4 4 4 6
40372023-09-22T23:25:25.846Z22/30 Write block 522 len 8192 data: 2 2
40382023-09-22T23:25:25.849Z23/30 Write block 443 len 4096 data: 4
40392023-09-22T23:25:25.853Z24/30 Read block 193 len 16384
40402023-09-22T23:25:25.855Z25/30 Write block 260 len 24576 data: 3 5 5 5 5 5
40412023-09-22T23:25:25.859Z26/30 Read block 168 len 28672
40422023-09-22T23:25:25.861Z27/30 Write block 7 len 12288 data: 4 4 4
40432023-09-22T23:25:25.864Z28/30 Read block 538 len 16384
40442023-09-22T23:25:25.867Z29/30 Read block 49 len 4096
40452023-09-22T23:25:25.870Z30/30 Write block 389 len 8192 data: 4 4
40462023-09-22T23:25:25.873Z----------------------------------------------------------------
40472023-09-22T23:25:25.876Z Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:30
40482023-09-22T23:25:25.879ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
40492023-09-22T23:25:25.882Z 1 Acked 1000 Write 6 Done Done Done false
40502023-09-22T23:25:25.885Z 2 Acked 1001 Read 2 Done Done Done false
40512023-09-22T23:25:25.887Z 3 Acked 1002 Write 7 Done Done Done false
40522023-09-22T23:25:25.890Z 4 Acked 1003 Read 1 Done Done Done false
40532023-09-22T23:25:25.893Z 5 Acked 1004 Write 9 Done Sent Done false
40542023-09-22T23:25:25.896Z 6 Acked 1005 Write 4 Done Sent Done false
40552023-09-22T23:25:25.899Z 7 Acked 1006 Read 3 Done Sent Done false
40562023-09-22T23:25:25.902Z 8 Acked 1007 Write 8 Done Sent Done false
40572023-09-22T23:25:25.905Z 9 Acked 1008 Write 3 Done Sent Done false
40582023-09-22T23:25:25.908Z 10 Acked 1009 Read 4 Done Sent Done false
40592023-09-22T23:25:25.910Z 11 Acked 1010 Write 3 Done Sent Done false
40602023-09-22T23:25:25.913Z 12 Acked 1011 Write 6 Done Sent Done false
40612023-09-22T23:25:25.916Z 13 Acked 1012 Read 8 Done Sent Done false
40622023-09-22T23:25:25.918Z 14 Acked 1013 Write 10 Done Sent Done false
40632023-09-22T23:25:25.921Z 15 Acked 1014 Read 4 Done Sent Done false
40642023-09-22T23:25:25.924Z 16 Acked 1015 Write 5 Done Sent Done false
40652023-09-22T23:25:25.927Z 17 Acked 1016 Read 6 Done Sent Done false
40662023-09-22T23:25:25.930Z 18 Acked 1017 Read 4 Done Sent Done false
40672023-09-22T23:25:25.933Z 19 Acked 1018 Flush 0 Done Sent Done false
40682023-09-22T23:25:25.936Z 20 Acked 1019 Write 9 Done Sent Done false
40692023-09-22T23:25:25.939Z 21 Acked 1020 Write 5 Done Sent Done false
40702023-09-22T23:25:25.942Z 22 Acked 1021 Write 2 Done Sent Done false
40712023-09-22T23:25:25.945Z 23 Acked 1022 Write 1 Done Sent Done false
40722023-09-22T23:25:25.948Z 24 Acked 1023 Read 4 Done Sent Done false
40732023-09-22T23:25:25.951Z 25 Acked 1024 Write 6 Done Sent Done false
40742023-09-22T23:25:25.953Z 26 Acked 1025 Read 7 Done Sent Done false
40752023-09-22T23:25:25.956Z 27 Acked 1026 Write 3 Done Sent Done false
40762023-09-22T23:25:25.959Z 28 Acked 1027 Read 4 Done Sent Done false
40772023-09-22T23:25:25.962Z 29 Acked 1028 Read 1 Done Sent Done false
40782023-09-22T23:25:25.965Z 30 Acked 1029 Write 2 Sent Sent Sent false
40792023-09-22T23:25:25.968Z STATES DS:0 DS:1 DS:2 TOTAL
40802023-09-22T23:25:25.971Z New 0 0 0 0
40812023-09-22T23:25:25.975Z Sent 1 26 1 28
40822023-09-22T23:25:25.978Z Done 29 4 29 62
40832023-09-22T23:25:25.981Z Skipped 0 0 0 0
40842023-09-22T23:25:25.984Z Error 0 0 0 0
40852023-09-22T23:25:25.987ZLast Flush: 1018 0 1018
40862023-09-22T23:25:25.990ZDownstairs last five completed:
40872023-09-22T23:25:25.993ZUpstairs last five completed: 30 29 28 27 26 25
40882023-09-22T23:25:25.996ZWrote out file "/tmp/test_repair_verify.data"
40892023-09-22T23:25:25.999Z
40902023-09-22T23:25:26.003ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
40912023-09-22T23:25:26.006Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.656396542Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1280}
40922023-09-22T23:25:26.009Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.656726128Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1280}
40932023-09-22T23:25:26.012Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.656749311Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1280}
40942023-09-22T23:25:26.015Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.68656658Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1280}
40952023-09-22T23:25:26.018Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.68660126Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1280}
40962023-09-22T23:25:26.020Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.686619987Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1280}
40972023-09-22T23:25:26.023Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.716399954Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1280}
40982023-09-22T23:25:26.026Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.716436213Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1280}
40992023-09-22T23:25:26.029Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:24.716455594Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1280}
41002023-09-22T23:25:26.032ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
41012023-09-22T23:25:26.035ZError: Difference in extent metadata found!
41022023-09-22T23:25:26.037Z 0 000-019 29 29 29 43 43 43 T F T <---
41032023-09-22T23:25:26.040Z 1 020-039 24 24 24 34 34 34 F F F
41042023-09-22T23:25:26.043Z 2 040-059 29 29 29 43 43 43 F F F
41052023-09-22T23:25:26.046Z 3 060-079 29 29 29 43 43 43 F F F
41062023-09-22T23:25:26.049Z 4 080-099 28 28 28 42 42 42 F F F
41072023-09-22T23:25:26.052Z 5 100-119 27 27 27 41 41 41 F F F
41082023-09-22T23:25:26.055Z 6 120-139 30 25 30 44 36 44 F T F <---
41092023-09-22T23:25:26.058Z 7 140-159 30 27 30 44 41 44 F T F <---
41102023-09-22T23:25:26.061Z 8 160-179 27 27 27 41 41 41 T F T <---
41112023-09-22T23:25:26.063Z 9 180-199 30 26 30 44 37 44 F F F <---
41122023-09-22T23:25:26.066Z 10 200-219 30 26 30 44 40 44 F F F <---
41132023-09-22T23:25:26.069Z 11 220-239 30 24 30 44 34 44 T F T <---
41142023-09-22T23:25:26.072Z 12 240-259 30 22 30 44 33 44 F F F <---
41152023-09-22T23:25:26.075Z 13 260-279 30 28 30 44 42 44 T F T <---
41162023-09-22T23:25:26.078Z 14 280-299 28 28 28 42 42 42 F F F
41172023-09-22T23:25:26.080Z 15 300-319 27 27 27 41 41 41 F F F
41182023-09-22T23:25:26.083Z 16 320-339 27 27 27 41 41 41 F F F
41192023-09-22T23:25:26.086Z 17 340-359 29 29 29 43 43 43 F F F
41202023-09-22T23:25:26.089Z 18 360-379 27 27 27 41 41 41 F F F
41212023-09-22T23:25:26.092Z 19 380-399 28 28 28 42 42 42 T F T <---
41222023-09-22T23:25:26.096Z 20 400-419 28 28 28 42 42 42 F F F
41232023-09-22T23:25:26.098Z 21 420-439 29 29 29 43 43 43 F F F
41242023-09-22T23:25:26.101Z 22 440-459 29 29 29 43 43 43 T F T <---
41252023-09-22T23:25:26.104Z 23 460-479 30 28 30 44 42 44 F F F <---
41262023-09-22T23:25:26.107Z 24 480-499 29 29 29 43 43 43 F F F
41272023-09-22T23:25:26.109Z 25 500-519 30 28 30 44 42 44 F F F <---
41282023-09-22T23:25:26.112Z 26 520-539 21 21 21 32 32 32 T F T <---
41292023-09-22T23:25:26.115Z 27 540-559 27 27 27 41 41 41 F F F
41302023-09-22T23:25:26.118Z 28 560-579 22 22 22 33 33 33 F F F
41312023-09-22T23:25:26.121Z 29 580-599 30 28 30 44 42 44 F T F <---
41322023-09-22T23:25:26.124ZMax gen: 30, Max flush: 44
41332023-09-22T23:25:26.126ZA Difference in extent metadata is expected here
41342023-09-22T23:25:26.129ZOn loop 15
41352023-09-22T23:25:26.132Z
41362023-09-22T23:25:26.135ZVerifying data now
41372023-09-22T23:25:32.419ZNo change after verify
41382023-09-22T23:25:32.422ZLoop: 15 Downstairs dump after verify (and repair):
41392023-09-22T23:25:32.437Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.436167766Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1285}
41402023-09-22T23:25:32.440Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.436515887Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1285}
41412023-09-22T23:25:32.443Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.43654092Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1285}
41422023-09-22T23:25:32.456Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.455332617Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1285}
41432023-09-22T23:25:32.459Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.455386291Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1285}
41442023-09-22T23:25:32.462Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.455406553Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1285}
41452023-09-22T23:25:32.467Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.466243977Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1285}
41462023-09-22T23:25:32.470Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.46629573Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1285}
41472023-09-22T23:25:32.473Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.466316098Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1285}
41482023-09-22T23:25:32.478ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
41492023-09-22T23:25:32.481Z 0 000-019 31 31 31 45 45 45 F F F
41502023-09-22T23:25:32.484Z 1 020-039 24 24 24 34 34 34 F F F
41512023-09-22T23:25:32.487Z 2 040-059 29 29 29 43 43 43 F F F
41522023-09-22T23:25:32.490Z 3 060-079 29 29 29 43 43 43 F F F
41532023-09-22T23:25:32.492Z 4 080-099 28 28 28 42 42 42 F F F
41542023-09-22T23:25:32.495Z 5 100-119 27 27 27 41 41 41 F F F
41552023-09-22T23:25:32.498Z 6 120-139 30 30 30 44 44 44 F F F
41562023-09-22T23:25:32.501Z 7 140-159 30 30 30 44 44 44 F F F
41572023-09-22T23:25:32.503Z 8 160-179 31 31 31 45 45 45 F F F
41582023-09-22T23:25:32.506Z 9 180-199 30 30 30 44 44 44 F F F
41592023-09-22T23:25:32.509Z 10 200-219 30 30 30 44 44 44 F F F
41602023-09-22T23:25:32.512Z 11 220-239 31 31 31 45 45 45 F F F
41612023-09-22T23:25:32.515Z 12 240-259 30 30 30 44 44 44 F F F
41622023-09-22T23:25:32.517Z 13 260-279 31 31 31 45 45 45 F F F
41632023-09-22T23:25:32.520Z 14 280-299 28 28 28 42 42 42 F F F
41642023-09-22T23:25:32.523Z 15 300-319 27 27 27 41 41 41 F F F
41652023-09-22T23:25:32.526Z 16 320-339 27 27 27 41 41 41 F F F
41662023-09-22T23:25:32.529Z 17 340-359 29 29 29 43 43 43 F F F
41672023-09-22T23:25:32.532Z 18 360-379 27 27 27 41 41 41 F F F
41682023-09-22T23:25:32.535Z 19 380-399 31 31 31 45 45 45 F F F
41692023-09-22T23:25:32.538Z 20 400-419 28 28 28 42 42 42 F F F
41702023-09-22T23:25:32.541Z 21 420-439 29 29 29 43 43 43 F F F
41712023-09-22T23:25:32.544Z 22 440-459 31 31 31 45 45 45 F F F
41722023-09-22T23:25:32.546Z 23 460-479 30 30 30 44 44 44 F F F
41732023-09-22T23:25:32.549Z 24 480-499 29 29 29 43 43 43 F F F
41742023-09-22T23:25:32.552Z 25 500-519 30 30 30 44 44 44 F F F
41752023-09-22T23:25:32.555Z 26 520-539 31 31 31 45 45 45 F F F
41762023-09-22T23:25:32.558Z 27 540-559 27 27 27 41 41 41 F F F
41772023-09-22T23:25:32.561Z 28 560-579 22 22 22 33 33 33 F F F
41782023-09-22T23:25:32.564Z 29 580-599 30 30 30 44 44 44 F F F
41792023-09-22T23:25:32.566ZMax gen: 31, Max flush: 45
41802023-09-22T23:25:32.569Z
41812023-09-22T23:25:32.572ZBegin loop 16
41822023-09-22T23:25:32.574ZDownstairs to restart: 2
41832023-09-22T23:25:32.577Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.517759805Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
41842023-09-22T23:25:32.580Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.518127337Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
41852023-09-22T23:25:32.583Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.518178937Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
41862023-09-22T23:25:32.587Z{"msg":"Crucible stats registered with UUID: 79b8fc53-c509-4a8d-ab40-fabb4ce1ebc9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.518223354Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
41872023-09-22T23:25:32.589Z{"msg":"Crucible 79b8fc53-c509-4a8d-ab40-fabb4ce1ebc9 has session id: 631a2c56-6608-4c7b-84ee-489efdb6c0a3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.518282229Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
41882023-09-22T23:25:32.592ZCrucible runtime is spawned
41892023-09-22T23:25:32.595Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.518484775Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287,"task":"up_listen"}
41902023-09-22T23:25:32.598ZThe guest has requested activation with gen:32
41912023-09-22T23:25:32.600Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.518546323Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
41922023-09-22T23:25:32.603Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.5185756Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
41932023-09-22T23:25:32.606Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.518648738Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287,"looper":"2"}
41942023-09-22T23:25:32.609Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.518969671Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287,"looper":"0"}
41952023-09-22T23:25:32.612Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.519201009Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287,"looper":"1"}
41962023-09-22T23:25:32.615Z{"msg":"[0] 79b8fc53-c509-4a8d-ab40-fabb4ce1ebc9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.519284056Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287,"looper":"0"}
41972023-09-22T23:25:32.617Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.519405722Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
41982023-09-22T23:25:32.620Z{"msg":"[1] 79b8fc53-c509-4a8d-ab40-fabb4ce1ebc9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.519487771Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287,"looper":"1"}
41992023-09-22T23:25:32.623Z{"msg":"79b8fc53-c509-4a8d-ab40-fabb4ce1ebc9 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.519623113Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42002023-09-22T23:25:32.626Z{"msg":"Set desired generation to :32","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.519723184Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42012023-09-22T23:25:32.629Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.519798485Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42022023-09-22T23:25:32.632Z{"msg":"[1] 79b8fc53-c509-4a8d-ab40-fabb4ce1ebc9 (0a2c8a5c-01b8-43c9-95e3-8d517f639007) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.52031105Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42032023-09-22T23:25:32.635Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.520367474Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42042023-09-22T23:25:32.638Z{"msg":"[0] 79b8fc53-c509-4a8d-ab40-fabb4ce1ebc9 (0a2c8a5c-01b8-43c9-95e3-8d517f639007) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.520428519Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42052023-09-22T23:25:32.641Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.520471092Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42062023-09-22T23:25:32.644Z{"msg":"[1] client is_active_req TRUE, promote! session 0a2c8a5c-01b8-43c9-95e3-8d517f639007","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.520508146Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42072023-09-22T23:25:32.647Z{"msg":"[0] client is_active_req TRUE, promote! session 0a2c8a5c-01b8-43c9-95e3-8d517f639007","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.520573239Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42082023-09-22T23:25:32.652Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.521139955Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42092023-09-22T23:25:32.655Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.521203961Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42102023-09-22T23:25:32.659Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.521250217Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42112023-09-22T23:25:32.661Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.521276607Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42122023-09-22T23:25:32.664Z{"msg":"79b8fc53-c509-4a8d-ab40-fabb4ce1ebc9 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.521314958Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42132023-09-22T23:25:32.667Z{"msg":"79b8fc53-c509-4a8d-ab40-fabb4ce1ebc9 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.5213383Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42142023-09-22T23:25:32.670Z{"msg":"[1] 79b8fc53-c509-4a8d-ab40-fabb4ce1ebc9 (0a2c8a5c-01b8-43c9-95e3-8d517f639007) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.523564934Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42152023-09-22T23:25:32.673Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.523722579Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42162023-09-22T23:25:32.675Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:32.523782036Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42172023-09-22T23:25:32.678Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.523838056Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42182023-09-22T23:25:32.681Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.523894305Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42192023-09-22T23:25:32.684Z{"msg":"79b8fc53-c509-4a8d-ab40-fabb4ce1ebc9 WaitActive WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.523950115Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42202023-09-22T23:25:32.686Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.52400564Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42212023-09-22T23:25:32.689Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.524061257Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42222023-09-22T23:25:32.692Z{"msg":"[0] 79b8fc53-c509-4a8d-ab40-fabb4ce1ebc9 (0a2c8a5c-01b8-43c9-95e3-8d517f639007) WaitActive WaitQuorum New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.524120053Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42232023-09-22T23:25:32.695Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.52417529Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42242023-09-22T23:25:32.698Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:32.524230652Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42252023-09-22T23:25:32.701Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.524286372Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42262023-09-22T23:25:32.704Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.524342261Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42272023-09-22T23:25:32.707Z{"msg":"79b8fc53-c509-4a8d-ab40-fabb4ce1ebc9 WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.524397434Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42282023-09-22T23:25:32.710Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.524452282Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42292023-09-22T23:25:32.712Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:32.524506989Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42302023-09-22T23:25:33.521Z{"msg":"[2] 79b8fc53-c509-4a8d-ab40-fabb4ce1ebc9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.52002655Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287,"looper":"2"}
42312023-09-22T23:25:33.524Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.520110685Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42322023-09-22T23:25:33.527Z{"msg":"[2] 79b8fc53-c509-4a8d-ab40-fabb4ce1ebc9 (0a2c8a5c-01b8-43c9-95e3-8d517f639007) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.521140234Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42332023-09-22T23:25:33.531Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.521195489Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42342023-09-22T23:25:33.534Z{"msg":"[2] client is_active_req TRUE, promote! session 0a2c8a5c-01b8-43c9-95e3-8d517f639007","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.521239098Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42352023-09-22T23:25:33.537Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.521837449Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42362023-09-22T23:25:33.540Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.52189374Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42372023-09-22T23:25:33.543Z{"msg":"79b8fc53-c509-4a8d-ab40-fabb4ce1ebc9 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.521936519Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42382023-09-22T23:25:33.546Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.524239389Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42392023-09-22T23:25:33.549Z{"msg":"[2] 79b8fc53-c509-4a8d-ab40-fabb4ce1ebc9 (0a2c8a5c-01b8-43c9-95e3-8d517f639007) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.524412627Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42402023-09-22T23:25:33.552Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.524486856Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42412023-09-22T23:25:33.555Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:33.524518679Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42422023-09-22T23:25:33.557Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.524559139Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42432023-09-22T23:25:33.561Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.524581943Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42442023-09-22T23:25:33.564Z{"msg":"79b8fc53-c509-4a8d-ab40-fabb4ce1ebc9 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.524619657Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42452023-09-22T23:25:33.567Z{"msg":"[0]R flush_numbers[0..12]: [45, 34, 43, 43, 42, 41, 44, 44, 45, 44, 44, 45]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.52464226Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42462023-09-22T23:25:33.570Z{"msg":"[0]R generation[0..12]: [31, 24, 29, 29, 28, 27, 30, 30, 31, 30, 30, 31]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.524679829Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42472023-09-22T23:25:33.572Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.524719664Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42482023-09-22T23:25:33.575Z{"msg":"[1]R flush_numbers[0..12]: [45, 34, 43, 43, 42, 41, 44, 44, 45, 44, 44, 45]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.524758871Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42492023-09-22T23:25:33.578Z{"msg":"[1]R generation[0..12]: [31, 24, 29, 29, 28, 27, 30, 30, 31, 30, 30, 31]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.524808408Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42502023-09-22T23:25:33.581Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.524832409Z"The guest has finished waiting for activation with:32
42512023-09-22T23:25:33.585Z,"hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42522023-09-22T23:25:33.587ZWait for a query_work_queue command to finish before sending IO
42532023-09-22T23:25:33.590Z{"msg":"[2]R flush_numbers[0..12]: [45, 34, 43, 43, 42, 41, 44, 44, 45, 44, 44, 45]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.524859462Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42542023-09-22T23:25:33.593Z{"msg":"[2]R generation[0..12]: [31, 24, 29, 29, 28, 27, 30, 30, 31, 30, 30, 31]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.524899408Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42552023-09-22T23:25:33.595Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.524920822Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42562023-09-22T23:25:33.598Z{"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.524958642Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42572023-09-22T23:25:33.601Z{"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.524982136Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42582023-09-22T23:25:33.604Z{"msg":"Next flush: 46","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.525038499Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42592023-09-22T23:25:33.607Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.525058624Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42602023-09-22T23:25:33.609Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.525095319Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42612023-09-22T23:25:33.612Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.525141384Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42622023-09-22T23:25:33.615Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.525187603Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42632023-09-22T23:25:33.618Z{"msg":"79b8fc53-c509-4a8d-ab40-fabb4ce1ebc9 is now active with session: 0a2c8a5c-01b8-43c9-95e3-8d517f639007","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.525276581Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42642023-09-22T23:25:33.621Z{"msg":"79b8fc53-c509-4a8d-ab40-fabb4ce1ebc9 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.525317092Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42652023-09-22T23:25:33.624Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.525355982Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42662023-09-22T23:25:33.627Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.525375063Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42672023-09-22T23:25:33.630Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.525411202Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42682023-09-22T23:25:33.632Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.525430811Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42692023-09-22T23:25:33.635Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.525466467Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42702023-09-22T23:25:33.638Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.525485723Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42712023-09-22T23:25:33.641Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.525521086Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42722023-09-22T23:25:33.644Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.525569401Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42732023-09-22T23:25:33.647Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.525588357Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42742023-09-22T23:25:33.654Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.525630522Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42752023-09-22T23:25:33.657Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.525649575Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1287}
42762023-09-22T23:25:33.660ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
42772023-09-22T23:25:33.663ZLoading write count information from file "/tmp/test_repair_verify.data"
42782023-09-22T23:25:33.665ZRun Repair workload
42792023-09-22T23:25:33.668Z01/30 Write block 426 len 36864 data: 2 4 4 4 4 4 6 5 4
42802023-09-22T23:25:33.671Z02/30 Write block 208 len 16384 data: 7 7 7 8
42812023-09-22T23:25:33.674Z03/30 Read block 0 len 32768
42822023-09-22T23:25:33.676Z04/30 Write block 497 len 16384 data: 4 4 4 5
42832023-09-22T23:25:33.679Z05/30 Read block 562 len 28672
42842023-09-22T23:25:33.682Z06/30 Read block 415 len 36864
42852023-09-22T23:25:33.685Z07/30 Read block 538 len 36864
42862023-09-22T23:25:33.688Z08/30 Write block 134 len 4096 data: 4
42872023-09-22T23:25:33.690Z09/30 Read block 474 len 24576
42882023-09-22T23:25:33.693Z10/30 Read block 282 len 16384
42892023-09-22T23:25:33.696Z11/30 Write block 241 len 20480 data: 3 3 4 2 2
42902023-09-22T23:25:33.699Z12/30 Write block 452 len 32768 data: 3 3 3 3 2 3 4 4
42912023-09-22T23:25:33.702Z13/30 Read block 264 len 4096
42922023-09-22T23:25:33.705Z14/30 Read block 426 len 40960
42932023-09-22T23:25:33.708Z15/30 Flush
42942023-09-22T23:25:33.711Z16/30 Write block 384 len 32768 data: 3 3 4 6 5 5 5 4
42952023-09-22T23:25:33.714Z17/30 Read block 278 len 12288
42962023-09-22T23:25:33.717Z18/30 Read block 535 len 4096
42972023-09-22T23:25:33.720Z19/30 Read block 287 len 16384
42982023-09-22T23:25:33.724Z20/30 Read block 292 len 40960
42992023-09-22T23:25:33.726Z21/30 Write block 278 len 8192 data: 5 5
43002023-09-22T23:25:33.729Z22/30 Read block 582 len 40960
43012023-09-22T23:25:33.732Z23/30 Write block 559 len 24576 data: 3 3 3 2 2 3
43022023-09-22T23:25:33.735Z24/30 Write block 274 len 36864 data: 5 5 7 4 6 6 4 3 4
43032023-09-22T23:25:33.738Z25/30 Write block 546 len 12288 data: 5 3 4
43042023-09-22T23:25:33.741Z26/30 Flush
43052023-09-22T23:25:33.744Z27/30 Write block 210 len 20480 data: 8 9 7 7 5
43062023-09-22T23:25:33.747Z28/30 Read block 386 len 20480
43072023-09-22T23:25:33.751Z29/30 Write block 262 len 32768 data: 6 6 6 6 5 6 4 5
43082023-09-22T23:25:33.755Z30/30 Read block 246 len 24576
43092023-09-22T23:25:33.758Z----------------------------------------------------------------
43102023-09-22T23:25:33.761Z Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30
43112023-09-22T23:25:33.767ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
43122023-09-22T23:25:33.770Z 1 Acked 1000 Write 9 Done Done Done false
43132023-09-22T23:25:33.773Z 2 Acked 1001 Write 4 Done Done Sent false
43142023-09-22T23:25:33.777Z 3 Acked 1002 Read 8 Done Done Sent false
43152023-09-22T23:25:33.780Z 4 Acked 1003 Write 4 Done Done Sent false
43162023-09-22T23:25:33.783Z 5 Acked 1004 Read 7 Done Done Sent false
43172023-09-22T23:25:33.787Z 6 Acked 1005 Read 9 Done Done Sent false
43182023-09-22T23:25:33.790Z 7 Acked 1006 Read 9 Done Done Sent false
43192023-09-22T23:25:33.793Z 8 Acked 1007 Write 1 Done Done Sent false
43202023-09-22T23:25:33.798Z 9 Acked 1008 Read 6 Done Done Sent false
43212023-09-22T23:25:33.821Z 10 Acked 1009 Read 4 Done Done Sent false
43222023-09-22T23:25:33.824Z 11 Acked 1010 Write 5 Done Done Sent false
43232023-09-22T23:25:33.829Z 12 Acked 1011 Write 8 Done Done Sent false
43242023-09-22T23:25:33.832Z 13 Acked 1012 Read 1 Done Done Sent false
43252023-09-22T23:25:33.835Z 14 Acked 1013 Read 10 Done Done Sent false
43262023-09-22T23:25:33.840Z 15 Acked 1014 Flush 0 Done Done Sent false
43272023-09-22T23:25:33.843Z 16 Acked 1015 Write 8 Done Done Sent false
43282023-09-22T23:25:33.847Z 17 Acked 1016 Read 3 Done Done Sent false
43292023-09-22T23:25:33.850Z 18 Acked 1017 Read 1 Done Done Sent false
43302023-09-22T23:25:33.854Z 19 Acked 1018 Read 4 Done Done Sent false
43312023-09-22T23:25:33.857Z 20 Acked 1019 Read 10 Done Done Sent false
43322023-09-22T23:25:33.861Z 21 Acked 1020 Write 2 Done Done Sent false
43332023-09-22T23:25:33.865Z 22 Acked 1021 Read 10 Done Done Sent false
43342023-09-22T23:25:33.868Z 23 Acked 1022 Write 6 Done Done Sent false
43352023-09-22T23:25:33.871Z 24 Acked 1023 Write 9 Done Done Sent false
43362023-09-22T23:25:33.874Z 25 Acked 1024 Write 3 Done Done Sent false
43372023-09-22T23:25:33.877Z 26 Acked 1025 Flush 0 Done Done Sent false
43382023-09-22T23:25:33.881Z 27 Acked 1026 Write 5 Done Done Sent false
43392023-09-22T23:25:33.885Z 28 Acked 1027 Read 5 Done Done Sent false
43402023-09-22T23:25:33.889Z 29 Acked 1028 Write 8 Done Done Sent false
43412023-09-22T23:25:33.892Z 30 Acked 1029 Read 6 Done Done Sent false
43422023-09-22T23:25:33.895Z STATES DS:0 DS:1 DS:2 TOTAL
43432023-09-22T23:25:33.900Z New 0 0 0 0
43442023-09-22T23:25:33.903Z Sent 0 0 29 29
43452023-09-22T23:25:33.907Z Done 30 30 1 61
43462023-09-22T23:25:33.911Z Skipped 0 0 0 0
43472023-09-22T23:25:33.915Z Error 0 0 0 0
43482023-09-22T23:25:33.918ZLast Flush: 1025 1025 0
43492023-09-22T23:25:33.921ZDownstairs last five completed:
43502023-09-22T23:25:33.925ZUpstairs last five completed: 30 29 28 27 26 25
43512023-09-22T23:25:33.928ZWrote out file "/tmp/test_repair_verify.data"
43522023-09-22T23:25:33.935Z
43532023-09-22T23:25:33.938ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
43542023-09-22T23:25:33.941Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.668605878Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1288}
43552023-09-22T23:25:33.945Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.668893434Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1288}
43562023-09-22T23:25:33.949Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.668916188Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1288}
43572023-09-22T23:25:33.952Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.691657883Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1288}
43582023-09-22T23:25:33.955Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.691691547Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1288}
43592023-09-22T23:25:33.959Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.69171084Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1288}
43602023-09-22T23:25:33.962Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.705431797Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1288}
43612023-09-22T23:25:33.966Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.705463403Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1288}
43622023-09-22T23:25:33.971Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:33.705482188Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1288}
43632023-09-22T23:25:33.974ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
43642023-09-22T23:25:33.976Z 0 000-019 31 31 31 45 45 45 F F F
43652023-09-22T23:25:33.979ZError: Difference in extent metadata found!
43662023-09-22T23:25:33.982Z 1 020-039 24 24 24 34 34 34 F F F
43672023-09-22T23:25:33.986Z 2 040-059 29 29 29 43 43 43 F F F
43682023-09-22T23:25:33.989Z 3 060-079 29 29 29 43 43 43 F F F
43692023-09-22T23:25:33.991Z 4 080-099 28 28 28 42 42 42 F F F
43702023-09-22T23:25:33.994Z 5 100-119 27 27 27 41 41 41 F F F
43712023-09-22T23:25:33.997Z 6 120-139 32 32 30 46 46 44 F F F <---
43722023-09-22T23:25:34.000Z 7 140-159 30 30 30 44 44 44 F F F
43732023-09-22T23:25:34.003Z 8 160-179 31 31 31 45 45 45 F F F
43742023-09-22T23:25:34.005Z 9 180-199 30 30 30 44 44 44 F F F
43752023-09-22T23:25:34.008Z 10 200-219 32 32 30 46 46 44 T T F <---
43762023-09-22T23:25:34.012Z 11 220-239 31 31 31 45 45 45 F F F
43772023-09-22T23:25:34.015Z 12 240-259 32 32 30 46 46 44 F F F <---
43782023-09-22T23:25:34.017Z 13 260-279 32 32 31 47 47 45 T T F <---
43792023-09-22T23:25:34.020Z 14 280-299 32 32 28 47 47 42 F F F <---
43802023-09-22T23:25:34.023Z 15 300-319 27 27 27 41 41 41 F F F
43812023-09-22T23:25:34.026Z 16 320-339 27 27 27 41 41 41 F F F
43822023-09-22T23:25:34.028Z 17 340-359 29 29 29 43 43 43 F F F
43832023-09-22T23:25:34.031Z 18 360-379 27 27 27 41 41 41 F F F
43842023-09-22T23:25:34.034Z 19 380-399 32 32 31 47 47 45 F F F <---
43852023-09-22T23:25:34.037Z 20 400-419 28 28 28 42 42 42 F F F
43862023-09-22T23:25:34.040Z 21 420-439 32 32 29 46 46 43 F F T <---
43872023-09-22T23:25:34.042Z 22 440-459 32 32 31 46 46 45 F F F <---
43882023-09-22T23:25:34.045Z 23 460-479 30 30 30 44 44 44 F F F
43892023-09-22T23:25:34.048Z 24 480-499 32 32 29 46 46 43 F F F <---
43902023-09-22T23:25:34.051Z 25 500-519 32 32 30 46 46 44 F F F <---
43912023-09-22T23:25:34.053Z 26 520-539 31 31 31 45 45 45 F F F
43922023-09-22T23:25:34.056Z 27 540-559 32 32 27 47 47 41 F F F <---
43932023-09-22T23:25:34.059Z 28 560-579 32 32 22 47 47 33 F F F <---
43942023-09-22T23:25:34.062Z 29 580-599 30 30 30 44 44 44 F F F
43952023-09-22T23:25:34.064ZMax gen: 32, Max flush: 47
43962023-09-22T23:25:34.067ZA Difference in extent metadata is expected here
43972023-09-22T23:25:34.070ZOn loop 16
43982023-09-22T23:25:34.072Z
43992023-09-22T23:25:34.075ZVerifying data now
44002023-09-22T23:25:40.066ZNo change after verify
44012023-09-22T23:25:40.069ZLoop: 16 Downstairs dump after verify (and repair):
44022023-09-22T23:25:40.084Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.082828326Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1293}
44032023-09-22T23:25:40.087Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.083257122Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1293}
44042023-09-22T23:25:40.090Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.083282291Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1293}
44052023-09-22T23:25:40.103Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.102381111Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1293}
44062023-09-22T23:25:40.106Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.102436209Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1293}
44072023-09-22T23:25:40.109Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.102456933Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1293}
44082023-09-22T23:25:40.114Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.113509455Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1293}
44092023-09-22T23:25:40.117Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.113563595Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1293}
44102023-09-22T23:25:40.120Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.113584435Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1293}
44112023-09-22T23:25:40.126ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
44122023-09-22T23:25:40.129Z 0 000-019 31 31 31 45 45 45 F F F
44132023-09-22T23:25:40.132Z 1 020-039 24 24 24 34 34 34 F F F
44142023-09-22T23:25:40.135Z 2 040-059 29 29 29 43 43 43 F F F
44152023-09-22T23:25:40.138Z 3 060-079 29 29 29 43 43 43 F F F
44162023-09-22T23:25:40.140Z 4 080-099 28 28 28 42 42 42 F F F
44172023-09-22T23:25:40.144Z 5 100-119 27 27 27 41 41 41 F F F
44182023-09-22T23:25:40.147Z 6 120-139 32 32 32 46 46 46 F F F
44192023-09-22T23:25:40.150Z 7 140-159 30 30 30 44 44 44 F F F
44202023-09-22T23:25:40.152Z 8 160-179 31 31 31 45 45 45 F F F
44212023-09-22T23:25:40.155Z 9 180-199 30 30 30 44 44 44 F F F
44222023-09-22T23:25:40.158Z 10 200-219 33 33 33 48 48 48 F F F
44232023-09-22T23:25:40.161Z 11 220-239 31 31 31 45 45 45 F F F
44242023-09-22T23:25:40.164Z 12 240-259 32 32 32 46 46 46 F F F
44252023-09-22T23:25:40.167Z 13 260-279 33 33 33 48 48 48 F F F
44262023-09-22T23:25:40.170Z 14 280-299 32 32 32 47 47 47 F F F
44272023-09-22T23:25:40.173Z 15 300-319 27 27 27 41 41 41 F F F
44282023-09-22T23:25:40.175Z 16 320-339 27 27 27 41 41 41 F F F
44292023-09-22T23:25:40.178Z 17 340-359 29 29 29 43 43 43 F F F
44302023-09-22T23:25:40.181Z 18 360-379 27 27 27 41 41 41 F F F
44312023-09-22T23:25:40.183Z 19 380-399 32 32 32 47 47 47 F F F
44322023-09-22T23:25:40.186Z 20 400-419 28 28 28 42 42 42 F F F
44332023-09-22T23:25:40.189Z 21 420-439 32 32 32 46 46 46 F F F
44342023-09-22T23:25:40.192Z 22 440-459 32 32 32 46 46 46 F F F
44352023-09-22T23:25:40.195Z 23 460-479 30 30 30 44 44 44 F F F
44362023-09-22T23:25:40.198Z 24 480-499 32 32 32 46 46 46 F F F
44372023-09-22T23:25:40.201Z 25 500-519 32 32 32 46 46 46 F F F
44382023-09-22T23:25:40.204Z 26 520-539 31 31 31 45 45 45 F F F
44392023-09-22T23:25:40.207Z 27 540-559 32 32 32 47 47 47 F F F
44402023-09-22T23:25:40.209Z 28 560-579 32 32 32 47 47 47 F F F
44412023-09-22T23:25:40.212Z 29 580-599 30 30 30 44 44 44 F F F
44422023-09-22T23:25:40.215ZMax gen: 33, Max flush: 48
44432023-09-22T23:25:40.218Z
44442023-09-22T23:25:40.221ZBegin loop 17
44452023-09-22T23:25:40.223ZDownstairs to restart: 0
44462023-09-22T23:25:40.226Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.164405823Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44472023-09-22T23:25:40.229Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.164781318Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44482023-09-22T23:25:40.232Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.164834675Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44492023-09-22T23:25:40.235Z{"msg":"Crucible stats registered with UUID: f960ffe3-208e-4c96-a7fc-a58e24970f92","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.164879227Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44502023-09-22T23:25:40.238Z{"msg":"Crucible f960ffe3-208e-4c96-a7fc-a58e24970f92 has session id: 354431f6-b7d1-4896-bd7c-095515cc2179","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.164925689Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44512023-09-22T23:25:40.241ZCrucible runtime is spawned
44522023-09-22T23:25:40.244Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.165148831Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295,"task":"up_listen"}
44532023-09-22T23:25:40.249Z{"msg":"The guest has requested activation with gen:34
44542023-09-22T23:25:40.252ZWait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.165232481Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44552023-09-22T23:25:40.254Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.165288546Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44562023-09-22T23:25:40.257Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.165330066Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295,"looper":"0"}
44572023-09-22T23:25:40.260Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.165571008Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295,"looper":"1"}
44582023-09-22T23:25:40.263Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.165859397Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295,"looper":"2"}
44592023-09-22T23:25:40.266Z{"msg":"[1] f960ffe3-208e-4c96-a7fc-a58e24970f92 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.165933862Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295,"looper":"1"}
44602023-09-22T23:25:40.269Z{"msg":"[2] f960ffe3-208e-4c96-a7fc-a58e24970f92 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.165997609Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295,"looper":"2"}
44612023-09-22T23:25:40.272Z{"msg":"f960ffe3-208e-4c96-a7fc-a58e24970f92 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.166418919Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44622023-09-22T23:25:40.275Z{"msg":"Set desired generation to :34","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.166479541Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44632023-09-22T23:25:40.277Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.166522945Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44642023-09-22T23:25:40.280Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.166841985Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44652023-09-22T23:25:40.283Z{"msg":"[1] f960ffe3-208e-4c96-a7fc-a58e24970f92 (699792b5-4a93-481a-a0ae-92921e692fe4) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.167161182Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44662023-09-22T23:25:40.285Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.167230157Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44672023-09-22T23:25:40.288Z{"msg":"[1] client is_active_req TRUE, promote! session 699792b5-4a93-481a-a0ae-92921e692fe4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.167302065Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44682023-09-22T23:25:40.291Z{"msg":"[2] f960ffe3-208e-4c96-a7fc-a58e24970f92 (699792b5-4a93-481a-a0ae-92921e692fe4) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.16737726Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44692023-09-22T23:25:40.294Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.167446096Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44702023-09-22T23:25:40.297Z{"msg":"[2] client is_active_req TRUE, promote! session 699792b5-4a93-481a-a0ae-92921e692fe4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.167500343Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44712023-09-22T23:25:40.300Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.167741909Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44722023-09-22T23:25:40.303Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.167806615Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44732023-09-22T23:25:40.306Z{"msg":"f960ffe3-208e-4c96-a7fc-a58e24970f92 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.167850679Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44742023-09-22T23:25:40.308Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.167902038Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44752023-09-22T23:25:40.311Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.167967119Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44762023-09-22T23:25:40.314Z{"msg":"f960ffe3-208e-4c96-a7fc-a58e24970f92 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.168039432Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44772023-09-22T23:25:40.317Z{"msg":"[1] f960ffe3-208e-4c96-a7fc-a58e24970f92 (699792b5-4a93-481a-a0ae-92921e692fe4) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.169999089Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44782023-09-22T23:25:40.320Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.170098587Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44792023-09-22T23:25:40.323Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:40.170171569Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44802023-09-22T23:25:40.325Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.170242968Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44812023-09-22T23:25:40.328Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.170307648Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44822023-09-22T23:25:40.331Z{"msg":"f960ffe3-208e-4c96-a7fc-a58e24970f92 New WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.170379288Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44832023-09-22T23:25:40.334Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.170453064Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44842023-09-22T23:25:40.338Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.170522225Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44852023-09-22T23:25:40.340Z{"msg":"[2] f960ffe3-208e-4c96-a7fc-a58e24970f92 (699792b5-4a93-481a-a0ae-92921e692fe4) New WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.170599834Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44862023-09-22T23:25:40.343Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.170662207Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44872023-09-22T23:25:40.346Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:40.170705656Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44882023-09-22T23:25:40.349Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.170744389Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44892023-09-22T23:25:40.352Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.170782776Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44902023-09-22T23:25:40.355Z{"msg":"f960ffe3-208e-4c96-a7fc-a58e24970f92 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.170821168Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44912023-09-22T23:25:40.358Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.170859847Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44922023-09-22T23:25:40.361Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:40.170896937Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44932023-09-22T23:25:41.168Z{"msg":"[0] f960ffe3-208e-4c96-a7fc-a58e24970f92 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.16663901Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295,"looper":"0"}
44942023-09-22T23:25:41.172Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.166727608Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44952023-09-22T23:25:41.175Z{"msg":"[0] f960ffe3-208e-4c96-a7fc-a58e24970f92 (699792b5-4a93-481a-a0ae-92921e692fe4) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.167870765Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44962023-09-22T23:25:41.178Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.167927364Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44972023-09-22T23:25:41.181Z{"msg":"[0] client is_active_req TRUE, promote! session 699792b5-4a93-481a-a0ae-92921e692fe4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.167970256Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44982023-09-22T23:25:41.184Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.168575898Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
44992023-09-22T23:25:41.187Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.16863751Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45002023-09-22T23:25:41.190Z{"msg":"f960ffe3-208e-4c96-a7fc-a58e24970f92 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.168680386Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45012023-09-22T23:25:41.193Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.170972971Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45022023-09-22T23:25:41.196Z{"msg":"[0] f960ffe3-208e-4c96-a7fc-a58e24970f92 (699792b5-4a93-481a-a0ae-92921e692fe4) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.171121022Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45032023-09-22T23:25:41.199Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.17119293Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45042023-09-22T23:25:41.202Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:41.171259535Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45052023-09-22T23:25:41.204Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.171303433Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45062023-09-22T23:25:41.207Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.171343037Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45072023-09-22T23:25:41.210Z{"msg":"f960ffe3-208e-4c96-a7fc-a58e24970f92 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.171382452Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45082023-09-22T23:25:41.213Z{"msg":"[0]R flush_numbers[0..12]: [45, 34, 43, 43, 42, 41, 46, 44, 45, 44, 48, 45]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.171422611Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45092023-09-22T23:25:41.216Z{"msg":"[0]R generation[0..12]: [31, 24, 29, 29, 28, 27, 32, 30, 31, 30, 33, 31]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.171462352Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45102023-09-22T23:25:41.219Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.171503245Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45112023-09-22T23:25:41.222Z{"msg":"[1]R flush_numbers[0..12]: [45, 34, 43, 43, 42, 41, 46, 44, 45, 44, 48, 45]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:34
45122023-09-22T23:25:41.225ZWait for a query_work_queue command to finish before sending IO
45132023-09-22T23:25:41.228Z,"time":"2023-09-22T23:25:41.171571433Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45142023-09-22T23:25:41.231Z{"msg":"[1]R generation[0..12]: [31, 24, 29, 29, 28, 27, 32, 30, 31, 30, 33, 31]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.171760243Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45152023-09-22T23:25:41.234Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.171835483Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45162023-09-22T23:25:41.237Z{"msg":"[2]R flush_numbers[0..12]: [45, 34, 43, 43, 42, 41, 46, 44, 45, 44, 48, 45]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.171882833Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45172023-09-22T23:25:41.240Z{"msg":"[2]R generation[0..12]: [31, 24, 29, 29, 28, 27, 32, 30, 31, 30, 33, 31]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.171923191Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45182023-09-22T23:25:41.243Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.171980891Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45192023-09-22T23:25:41.246Z{"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.172015925Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45202023-09-22T23:25:41.249Z{"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.172058772Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45212023-09-22T23:25:41.252Z{"msg":"Next flush: 49","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.172097495Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45222023-09-22T23:25:41.255Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.17213577Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45232023-09-22T23:25:41.258Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.172188089Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45242023-09-22T23:25:41.261Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.172242018Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45252023-09-22T23:25:41.264Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.172282675Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45262023-09-22T23:25:41.267Z{"msg":"f960ffe3-208e-4c96-a7fc-a58e24970f92 is now active with session: 699792b5-4a93-481a-a0ae-92921e692fe4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.172323486Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45272023-09-22T23:25:41.270Z{"msg":"f960ffe3-208e-4c96-a7fc-a58e24970f92 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.172379714Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45282023-09-22T23:25:41.273Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.172425116Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45292023-09-22T23:25:41.276Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.172468448Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45302023-09-22T23:25:41.279Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.172532364Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
45312023-09-22T23:25:41.281Zip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45322023-09-22T23:25:41.284Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.172639415Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45332023-09-22T23:25:41.287Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.172683822Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45342023-09-22T23:25:41.290Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.172722152Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45352023-09-22T23:25:41.293Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.172760274Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45362023-09-22T23:25:41.296Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.172798207Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45372023-09-22T23:25:41.299Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.172835634Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45382023-09-22T23:25:41.302Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.172873151Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45392023-09-22T23:25:41.305Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.172910398Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1295}
45402023-09-22T23:25:41.308ZLoading write count information from file "/tmp/test_repair_verify.data"
45412023-09-22T23:25:41.311ZRun Repair workload
45422023-09-22T23:25:41.313Z01/30 Write block 536 len 8192 data: 3 6
45432023-09-22T23:25:41.316Z02/30 Write block 225 len 8192 data: 4 5
45442023-09-22T23:25:41.319Z03/30 Read block 363 len 4096
45452023-09-22T23:25:41.322Z04/30 Read block 506 len 24576
45462023-09-22T23:25:41.325Z05/30 Write block 548 len 40960 data: 5 5 5 5 5 5 3 3 4 4
45472023-09-22T23:25:41.328Z06/30 Write block 484 len 12288 data: 4 4 4
45482023-09-22T23:25:41.330Z07/30 Flush
45492023-09-22T23:25:41.333Z08/30 Write block 454 len 28672 data: 4 4 3 4 5 5 4
45502023-09-22T23:25:41.336Z09/30 Write block 376 len 40960 data: 3 4 5 5 5 4 3 3 4 4
45512023-09-22T23:25:41.339Z10/30 Write block 142 len 28672 data: 5 5 4 3 2 2 2
45522023-09-22T23:25:41.342Z11/30 Read block 540 len 32768
45532023-09-22T23:25:41.345Z12/30 Read block 508 len 36864
45542023-09-22T23:25:41.348Z13/30 Read block 113 len 36864
45552023-09-22T23:25:41.351Z14/30 Flush
45562023-09-22T23:25:41.354Z15/30 Write block 298 len 28672 data: 3 3 3 2 2 2 2
45572023-09-22T23:25:41.356Z16/30 Write block 238 len 36864 data: 2 3 3 4 4 5 3 3 2
45582023-09-22T23:25:41.359Z17/30 Read block 143 len 8192
45592023-09-22T23:25:41.362Z18/30 Flush
45602023-09-22T23:25:41.365Z19/30 Write block 299 len 40960 data: 4 4 3 3 3 3 2 2 2 2
45612023-09-22T23:25:41.368Z20/30 Write block 249 len 36864 data: 4 4 3 5 4 4 3 3 3
45622023-09-22T23:25:41.371Z21/30 Write block 266 len 32768 data: 6 7 5 6 6 6 6 6
45632023-09-22T23:25:41.374Z22/30 Read block 584 len 16384
45642023-09-22T23:25:41.377Z23/30 Write block 58 len 36864 data: 4 4 5 5 5 6 6 6 6
45652023-09-22T23:25:41.380Z24/30 Read block 105 len 16384
45662023-09-22T23:25:41.383Z25/30 Read block 228 len 16384
45672023-09-22T23:25:41.386Z26/30 Write block 346 len 16384 data: 5 6 5 3
45682023-09-22T23:25:41.389Z27/30 Write block 324 len 12288 data: 5 4 4
45692023-09-22T23:25:41.392Z28/30 Read block 536 len 36864
45702023-09-22T23:25:41.395Z29/30 Write block 223 len 4096 data: 3
45712023-09-22T23:25:41.398Z30/30 Read block 382 len 36864
45722023-09-22T23:25:41.401Z----------------------------------------------------------------
45732023-09-22T23:25:41.403Z Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30
45742023-09-22T23:25:41.407ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
45752023-09-22T23:25:41.410Z 1 Acked 1000 Write 2 Done Done Done false
45762023-09-22T23:25:41.412Z 2 Acked 1001 Write 2 Done Done Done false
45772023-09-22T23:25:41.416Z 3 Acked 1002 Read 1 Done Done Done false
45782023-09-22T23:25:41.419Z 4 Acked 1003 Read 6 Sent Done Done false
45792023-09-22T23:25:41.422Z 5 Acked 1004 Write 10 Sent Done Done false
45802023-09-22T23:25:41.424Z 6 Acked 1005 Write 3 Sent Done Done false
45812023-09-22T23:25:41.428Z 7 Acked 1006 Flush 0 Sent Done Done false
45822023-09-22T23:25:41.431Z 8 Acked 1007 Write 7 Sent Done Done false
45832023-09-22T23:25:41.434Z 9 Acked 1008 Write 10 Sent Done Done false
45842023-09-22T23:25:41.436Z 10 Acked 1009 Write 7 Sent Done Done false
45852023-09-22T23:25:41.439Z 11 Acked 1010 Read 8 Sent Done Done false
45862023-09-22T23:25:41.442Z 12 Acked 1011 Read 9 Sent Done Done false
45872023-09-22T23:25:41.446Z 13 Acked 1012 Read 9 Sent Done Done false
45882023-09-22T23:25:41.448Z 14 Acked 1013 Flush 0 Sent Done Done false
45892023-09-22T23:25:41.453Z 15 Acked 1014 Write 7 Sent Done Done false
45902023-09-22T23:25:41.457Z 16 Acked 1015 Write 9 Sent Done Done false
45912023-09-22T23:25:41.461Z 17 Acked 1016 Read 2 Sent Done Done false
45922023-09-22T23:25:41.465Z 18 Acked 1017 Flush 0 Sent Done Done false
45932023-09-22T23:25:41.468Z 19 Acked 1018 Write 10 Sent Done Done false
45942023-09-22T23:25:41.471Z 20 Acked 1019 Write 9 Sent Done Done false
45952023-09-22T23:25:41.473Z 21 Acked 1020 Write 8 Sent Done Done false
45962023-09-22T23:25:41.476Z 22 Acked 1021 Read 4 Sent Done Done false
45972023-09-22T23:25:41.479Z 23 Acked 1022 Write 9 Sent Done Done false
45982023-09-22T23:25:41.482Z 24 Acked 1023 Read 4 Sent Done Done false
45992023-09-22T23:25:41.485Z 25 Acked 1024 Read 4 Sent Done Done false
46002023-09-22T23:25:41.488Z 26 Acked 1025 Write 4 Sent Done Done false
46012023-09-22T23:25:41.491Z 27 Acked 1026 Write 3 Sent Done Done false
46022023-09-22T23:25:41.494Z 28 Acked 1027 Read 9 Sent Done Done false
46032023-09-22T23:25:41.496Z 29 Acked 1028 Write 1 Sent Done Done false
46042023-09-22T23:25:41.499Z 30 Acked 1029 Read 9 Sent Done Done false
46052023-09-22T23:25:41.502Z STATES DS:0 DS:1 DS:2 TOTAL
46062023-09-22T23:25:41.505Z New 0 0 0 0
46072023-09-22T23:25:41.508Z Sent 27 0 0 27
46082023-09-22T23:25:41.511Z Done 3 30 30 63
46092023-09-22T23:25:41.513Z Skipped 0 0 0 0
46102023-09-22T23:25:41.516Z Error 0 0 0 0
46112023-09-22T23:25:41.519ZLast Flush: 0 1017 1017
46122023-09-22T23:25:41.521ZDownstairs last five completed:
46132023-09-22T23:25:41.524ZUpstairs last five completed: 30 29 28 27 26 25
46142023-09-22T23:25:41.527ZWrote out file "/tmp/test_repair_verify.data"
46152023-09-22T23:25:41.530Z
46162023-09-22T23:25:41.532ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
46172023-09-22T23:25:41.536Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.327164141Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1296}
46182023-09-22T23:25:41.539Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.327432135Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1296}
46192023-09-22T23:25:41.542Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.327455339Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1296}
46202023-09-22T23:25:41.545Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.359632447Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1296}
46212023-09-22T23:25:41.547Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.359687071Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1296}
46222023-09-22T23:25:41.550Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.359725954Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1296}
46232023-09-22T23:25:41.553Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.383509071Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1296}
46242023-09-22T23:25:41.556Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.383563666Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1296}
46252023-09-22T23:25:41.559Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:41.383602439Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1296}
46262023-09-22T23:25:41.562ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
46272023-09-22T23:25:41.565Z 0 000-019 31 31 31 45 45 45 F F F
46282023-09-22T23:25:41.567Z 1 020-039 24 24 24 34 34 34 F F F
46292023-09-22T23:25:41.570Z 2 040-059 29 29 29 43 43 43 F T T <---
46302023-09-22T23:25:41.574Z 3 060-079 29 29 29 43 43 43 F T T <---
46312023-09-22T23:25:41.577Z 4 080-099 28 28 28 42 42 42 F F F
46322023-09-22T23:25:41.580Z 5 100-119 27 27 27 41 41 41 F F F
46332023-09-22T23:25:41.583Z 6 120-139 32 32 32 46 46 46 F F F
46342023-09-22T23:25:41.586Z 7 140-159 30 34 34 44 50 50 F F F <---
46352023-09-22T23:25:41.589Z 8 160-179 31 31 31 45 45 45 F F F
46362023-09-22T23:25:41.592Z 9 180-199 30 30 30 44 44 44 F F F
46372023-09-22T23:25:41.595Z 10 200-219 33 33 33 48 48 48 F F F
46382023-09-22T23:25:41.597Z 11 220-239 31 34 34 45 51 51 T T T <---
46392023-09-22T23:25:41.600Z 12 240-259 32 34 34 46 51 51 F T T <---
46402023-09-22T23:25:41.604ZError: Difference in extent metadata found!
46412023-09-22T23:25:41.606Z 13 260-279 33 33 33 48 48 48 F T T <---
46422023-09-22T23:25:41.609Z 14 280-299 32 34 34 47 51 51 F T T <---
46432023-09-22T23:25:41.612Z 15 300-319 27 34 34 41 51 51 F T T <---
46442023-09-22T23:25:41.615Z 16 320-339 27 27 27 41 41 41 F T T <---
46452023-09-22T23:25:41.618Z 17 340-359 29 29 29 43 43 43 F T T <---
46462023-09-22T23:25:41.621Z 18 360-379 27 34 34 41 50 50 F F F <---
46472023-09-22T23:25:41.624Z 19 380-399 32 34 34 47 50 50 F F F <---
46482023-09-22T23:25:41.626Z 20 400-419 28 28 28 42 42 42 F F F
46492023-09-22T23:25:41.629Z 21 420-439 32 32 32 46 46 46 F F F
46502023-09-22T23:25:41.632Z 22 440-459 32 34 34 46 50 50 F F F <---
46512023-09-22T23:25:41.635Z 23 460-479 30 34 34 44 50 50 F F F <---
46522023-09-22T23:25:41.638Z 24 480-499 32 34 34 46 49 49 F F F <---
46532023-09-22T23:25:41.640Z 25 500-519 32 32 32 46 46 46 F F F
46542023-09-22T23:25:41.644Z 26 520-539 31 34 34 45 49 49 T F F <---
46552023-09-22T23:25:41.647Z 27 540-559 32 34 34 47 49 49 F F F <---
46562023-09-22T23:25:41.650Z 28 560-579 32 32 32 47 47 47 F F F
46572023-09-22T23:25:41.653Z 29 580-599 30 30 30 44 44 44 F F F
46582023-09-22T23:25:41.656ZMax gen: 34, Max flush: 51
46592023-09-22T23:25:41.659ZA Difference in extent metadata is expected here
46602023-09-22T23:25:41.662ZOn loop 17
46612023-09-22T23:25:41.665Z
46622023-09-22T23:25:41.668ZVerifying data now
46632023-09-22T23:25:48.158ZNo change after verify
46642023-09-22T23:25:48.161ZLoop: 17 Downstairs dump after verify (and repair):
46652023-09-22T23:25:48.176Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.174917786Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1301}
46662023-09-22T23:25:48.179Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.175320243Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1301}
46672023-09-22T23:25:48.182Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.175345576Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1301}
46682023-09-22T23:25:48.195Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.194360058Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1301}
46692023-09-22T23:25:48.198Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.19441293Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1301}
46702023-09-22T23:25:48.201Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.194433886Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1301}
46712023-09-22T23:25:48.206Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.205540464Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1301}
46722023-09-22T23:25:48.209Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.205592178Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1301}
46732023-09-22T23:25:48.212Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.205612466Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1301}
46742023-09-22T23:25:48.217ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
46752023-09-22T23:25:48.221Z 0 000-019 31 31 31 45 45 45 F F F
46762023-09-22T23:25:48.224Z 1 020-039 24 24 24 34 34 34 F F F
46772023-09-22T23:25:48.227Z 2 040-059 35 35 35 52 52 52 F F F
46782023-09-22T23:25:48.229Z 3 060-079 35 35 35 52 52 52 F F F
46792023-09-22T23:25:48.232Z 4 080-099 28 28 28 42 42 42 F F F
46802023-09-22T23:25:48.235Z 5 100-119 27 27 27 41 41 41 F F F
46812023-09-22T23:25:48.238Z 6 120-139 32 32 32 46 46 46 F F F
46822023-09-22T23:25:48.241Z 7 140-159 34 34 34 50 50 50 F F F
46832023-09-22T23:25:48.243Z 8 160-179 31 31 31 45 45 45 F F F
46842023-09-22T23:25:48.246Z 9 180-199 30 30 30 44 44 44 F F F
46852023-09-22T23:25:48.248Z 10 200-219 33 33 33 48 48 48 F F F
46862023-09-22T23:25:48.251Z 11 220-239 35 35 35 52 52 52 F F F
46872023-09-22T23:25:48.254Z 12 240-259 35 35 35 52 52 52 F F F
46882023-09-22T23:25:48.257Z 13 260-279 35 35 35 52 52 52 F F F
46892023-09-22T23:25:48.259Z 14 280-299 35 35 35 52 52 52 F F F
46902023-09-22T23:25:48.262Z 15 300-319 35 35 35 52 52 52 F F F
46912023-09-22T23:25:48.265Z 16 320-339 35 35 35 52 52 52 F F F
46922023-09-22T23:25:48.268Z 17 340-359 35 35 35 52 52 52 F F F
46932023-09-22T23:25:48.270Z 18 360-379 34 34 34 50 50 50 F F F
46942023-09-22T23:25:48.273Z 19 380-399 34 34 34 50 50 50 F F F
46952023-09-22T23:25:48.276Z 20 400-419 28 28 28 42 42 42 F F F
46962023-09-22T23:25:48.278Z 21 420-439 32 32 32 46 46 46 F F F
46972023-09-22T23:25:48.281Z 22 440-459 34 34 34 50 50 50 F F F
46982023-09-22T23:25:48.284Z 23 460-479 34 34 34 50 50 50 F F F
46992023-09-22T23:25:48.286Z 24 480-499 34 34 34 49 49 49 F F F
47002023-09-22T23:25:48.292Z 25 500-519 32 32 32 46 46 46 F F F
47012023-09-22T23:25:48.294Z 26 520-539 34 34 34 49 49 49 F F F
47022023-09-22T23:25:48.297Z 27 540-559 34 34 34 49 49 49 F F F
47032023-09-22T23:25:48.300Z 28 560-579 32 32 32 47 47 47 F F F
47042023-09-22T23:25:48.302Z 29 580-599 30 30 30 44 44 44 F F F
47052023-09-22T23:25:48.305ZMax gen: 35, Max flush: 52
47062023-09-22T23:25:48.308Z
47072023-09-22T23:25:48.311ZBegin loop 18
47082023-09-22T23:25:48.314ZDownstairs to restart: 1
47092023-09-22T23:25:48.316Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.254836555Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47102023-09-22T23:25:48.319Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.255219407Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47112023-09-22T23:25:48.321Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.255291929Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47122023-09-22T23:25:48.324ZCrucible runtime is spawned
47132023-09-22T23:25:48.327Z{"msg":"Crucible stats registered with UUID: 8107177a-8026-43e8-86fb-299ff8acea67","v":0,"name":"crucible","level":The guest has requested activation with gen:36
47142023-09-22T23:25:48.329Z30,"time":"2023-09-22T23:25:48.255631717Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47152023-09-22T23:25:48.332Z{"msg":"Crucible 8107177a-8026-43e8-86fb-299ff8acea67 has session id: 2fa13431-f256-4204-ad1c-775b7d5d7ec1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.255724056Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47162023-09-22T23:25:48.335Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.25580304Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303,"task":"up_listen"}
47172023-09-22T23:25:48.338Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.255886761Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47182023-09-22T23:25:48.341Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.25599277Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47192023-09-22T23:25:48.343Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.256067778Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303,"looper":"2"}
47202023-09-22T23:25:48.346Z{"msg":"[2] 8107177a-8026-43e8-86fb-299ff8acea67 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.256225359Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303,"looper":"2"}
47212023-09-22T23:25:48.348Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.256299572Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303,"looper":"1"}
47222023-09-22T23:25:48.351Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.256378376Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303,"looper":"0"}
47232023-09-22T23:25:48.354Z{"msg":"[0] 8107177a-8026-43e8-86fb-299ff8acea67 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.256494832Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303,"looper":"0"}
47242023-09-22T23:25:48.357Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.256568322Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47252023-09-22T23:25:48.359Z{"msg":"8107177a-8026-43e8-86fb-299ff8acea67 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.256612838Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47262023-09-22T23:25:48.362Z{"msg":"Set desired generation to :36","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.256651368Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47272023-09-22T23:25:48.365Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.256880532Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47282023-09-22T23:25:48.368Z{"msg":"[2] 8107177a-8026-43e8-86fb-299ff8acea67 (cac4a346-2ec5-4a0c-a5bf-23e2127f79e6) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.257252798Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47292023-09-22T23:25:48.371Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.257314924Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47302023-09-22T23:25:48.374Z{"msg":"[2] client is_active_req TRUE, promote! session cac4a346-2ec5-4a0c-a5bf-23e2127f79e6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.257359417Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47312023-09-22T23:25:48.377Z{"msg":"[0] 8107177a-8026-43e8-86fb-299ff8acea67 (cac4a346-2ec5-4a0c-a5bf-23e2127f79e6) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.257408961Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47322023-09-22T23:25:48.379Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.257455635Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47332023-09-22T23:25:48.382Z{"msg":"[0] client is_active_req TRUE, promote! session cac4a346-2ec5-4a0c-a5bf-23e2127f79e6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.257496451Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47342023-09-22T23:25:48.385Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.257868932Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47352023-09-22T23:25:48.388Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.257931113Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47362023-09-22T23:25:48.391Z{"msg":"8107177a-8026-43e8-86fb-299ff8acea67 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.257973637Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47372023-09-22T23:25:48.393Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.258072116Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47382023-09-22T23:25:48.396Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.258127108Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47392023-09-22T23:25:48.399Z{"msg":"8107177a-8026-43e8-86fb-299ff8acea67 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.258151166Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47402023-09-22T23:25:48.402Z{"msg":"[0] 8107177a-8026-43e8-86fb-299ff8acea67 (cac4a346-2ec5-4a0c-a5bf-23e2127f79e6) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.260475437Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47412023-09-22T23:25:48.405Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.260535128Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47422023-09-22T23:25:48.407Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:48.260577849Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47432023-09-22T23:25:48.410Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.260632062Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47442023-09-22T23:25:48.413Z{"msg":"[2] 8107177a-8026-43e8-86fb-299ff8acea67 (cac4a346-2ec5-4a0c-a5bf-23e2127f79e6) WaitQuorum New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.260717201Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47452023-09-22T23:25:48.415Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.260751546Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47462023-09-22T23:25:48.418Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:48.260803675Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47472023-09-22T23:25:48.421Z{"msg":"8107177a-8026-43e8-86fb-299ff8acea67 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.260860394Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47482023-09-22T23:25:48.424Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.260914007Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47492023-09-22T23:25:48.426Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.26096845Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47502023-09-22T23:25:48.429Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.261024524Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47512023-09-22T23:25:48.432Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.26107869Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47522023-09-22T23:25:48.434Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.261133283Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47532023-09-22T23:25:48.437Z{"msg":"8107177a-8026-43e8-86fb-299ff8acea67 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.261188846Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47542023-09-22T23:25:48.440Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.261244562Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47552023-09-22T23:25:48.443Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:48.261298309Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47562023-09-22T23:25:49.260Z{"msg":"[1] 8107177a-8026-43e8-86fb-299ff8acea67 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.258373449Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303,"looper":"1"}
47572023-09-22T23:25:49.263Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.258487255Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47582023-09-22T23:25:49.266Z{"msg":"[1] 8107177a-8026-43e8-86fb-299ff8acea67 (cac4a346-2ec5-4a0c-a5bf-23e2127f79e6) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.259677326Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47592023-09-22T23:25:49.270Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.259738728Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47602023-09-22T23:25:49.273Z{"msg":"[1] client is_active_req TRUE, promote! session cac4a346-2ec5-4a0c-a5bf-23e2127f79e6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.259782732Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47612023-09-22T23:25:49.276Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.260358697Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47622023-09-22T23:25:49.279Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.260420502Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47632023-09-22T23:25:49.282Z{"msg":"8107177a-8026-43e8-86fb-299ff8acea67 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.260463688Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47642023-09-22T23:25:49.285Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.262696598Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47652023-09-22T23:25:49.287Z{"msg":"[1] 8107177a-8026-43e8-86fb-299ff8acea67 (cac4a346-2ec5-4a0c-a5bf-23e2127f79e6) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.262943622Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47662023-09-22T23:25:49.290Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.263017823Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47672023-09-22T23:25:49.293Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:49.263040737Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47682023-09-22T23:25:49.296Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.263059725Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47692023-09-22T23:25:49.299Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.263098276Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47702023-09-22T23:25:49.302Z{"msg":"8107177a-8026-43e8-86fb-299ff8acea67 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.263138253Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47712023-09-22T23:25:49.305Z{"msg":"[0]R flush_numbers[0..12]: [45, 34, 52, 52, 42, 41, 46, 50, 45, 44, 48, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.263179674Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47722023-09-22T23:25:49.308Z{"msg":"[0]R generation[0..12]: [31, 24, 35, 35, 28, 27, 32, 34, 31, 30, 33, 35]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.263221456Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47732023-09-22T23:25:49.311Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.263261854Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47742023-09-22T23:25:49.313Z{"msg":"[1]R flush_numbers[0..12]: [45, 34, 52, 52, 42, 41, 46, 50, 45, 44, 48, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.263301956Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47752023-09-22T23:25:49.316ZThe guest has finished waiting for activation with:36
47762023-09-22T23:25:49.319Z{"msg":"[1]R generation[0..12]: [31, 24, 35, 35, 28, 27, 32, 34, 31, 30, 33, 35]Wait for a query_work_queue command to finish before sending IO
47772023-09-22T23:25:49.322Z","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.263352198Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47782023-09-22T23:25:49.324Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.263393003Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47792023-09-22T23:25:49.327Z{"msg":"[2]R flush_numbers[0..12]: [45, 34, 52, 52, 42, 41, 46, 50, 45, 44, 48, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.26344118Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47802023-09-22T23:25:49.330Z{"msg":"[2]R generation[0..12]: [31, 24, 35, 35, 28, 27, 32, 34, 31, 30, 33, 35]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.263497196Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47812023-09-22T23:25:49.333Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.263568071Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47822023-09-22T23:25:49.337Z{"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.26363674Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47832023-09-22T23:25:49.340Z{"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.26368945Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47842023-09-22T23:25:49.343Z{"msg":"Next flush: 53","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.263735162Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47852023-09-22T23:25:49.346Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.263799447Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47862023-09-22T23:25:49.348Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.263868169Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47872023-09-22T23:25:49.351Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.263999676Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47882023-09-22T23:25:49.354Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.264069541Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47892023-09-22T23:25:49.357Z{"msg":"8107177a-8026-43e8-86fb-299ff8acea67 is now active with session: cac4a346-2ec5-4a0c-a5bf-23e2127f79e6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.264139749Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47902023-09-22T23:25:49.360Z{"msg":"8107177a-8026-43e8-86fb-299ff8acea67 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.264210435Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47912023-09-22T23:25:49.363Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.264279976Z"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
47922023-09-22T23:25:49.366Z,"hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47932023-09-22T23:25:49.369Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.264396912Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47942023-09-22T23:25:49.372Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.264445114Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47952023-09-22T23:25:49.375Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.264484618Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47962023-09-22T23:25:49.378Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.264522776Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47972023-09-22T23:25:49.381Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.264561248Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47982023-09-22T23:25:49.384Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.264599364Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
47992023-09-22T23:25:49.387Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.26463673Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
48002023-09-22T23:25:49.390Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.264674709Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
48012023-09-22T23:25:49.393Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.264712712Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
48022023-09-22T23:25:49.395Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.264750338Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1303}
48032023-09-22T23:25:49.398ZLoading write count information from file "/tmp/test_repair_verify.data"
48042023-09-22T23:25:49.401ZRun Repair workload
48052023-09-22T23:25:49.404Z01/30 Write block 313 len 36864 data: 3 5 6 7 7 6 6 7 7
48062023-09-22T23:25:49.407Z02/30 Read block 592 len 24576
48072023-09-22T23:25:49.410Z03/30 Write block 358 len 28672 data: 4 3 4 3 3 3 3
48082023-09-22T23:25:49.413Z04/30 Write block 430 len 40960 data: 5 5 7 6 5 4 4 4 4 2
48092023-09-22T23:25:49.415Z05/30 Read block 420 len 40960
48102023-09-22T23:25:49.418Z06/30 Read block 58 len 12288
48112023-09-22T23:25:49.421Z07/30 Write block 403 len 36864 data: 4 4 4 4 4 4 5 4 5
48122023-09-22T23:25:49.424Z08/30 Read block 129 len 8192
48132023-09-22T23:25:49.427Z09/30 Write block 475 len 24576 data: 3 3 4 4 4 3
48142023-09-22T23:25:49.430Z10/30 Read block 425 len 8192
48152023-09-22T23:25:49.433Z11/30 Flush
48162023-09-22T23:25:49.435Z12/30 Write block 158 len 28672 data: 4 5 6 6 5 4 4
48172023-09-22T23:25:49.438Z13/30 Read block 364 len 36864
48182023-09-22T23:25:49.441Z14/30 Write block 358 len 24576 data: 5 4 5 4 4 4
48192023-09-22T23:25:49.444Z15/30 Read block 432 len 8192
48202023-09-22T23:25:49.447Z16/30 Read block 353 len 12288
48212023-09-22T23:25:49.449Z17/30 Write block 172 len 4096 data: 4
48222023-09-22T23:25:49.452Z18/30 Write block 584 len 8192 data: 4 3
48232023-09-22T23:25:49.455Z19/30 Flush
48242023-09-22T23:25:49.458Z20/30 Write block 215 len 16384 data: 6 5 5 5
48252023-09-22T23:25:49.460Z21/30 Write block 276 len 32768 data: 8 5 7 7 5 4 5 5
48262023-09-22T23:25:49.463Z22/30 Read block 191 len 4096
48272023-09-22T23:25:49.466Z23/30 Write block 127 len 16384 data: 5 4 3 4
48282023-09-22T23:25:49.469Z24/30 Write block 540 len 12288 data: 5 4 5
48292023-09-22T23:25:49.472Z25/30 Write block 290 len 16384 data: 6 6 5 5
48302023-09-22T23:25:49.475Z26/30 Write block 98 len 16384 data: 5 3 2 2
48312023-09-22T23:25:49.477Z27/30 Read block 437 len 40960
48322023-09-22T23:25:49.480Z28/30 Write block 110 len 28672 data: 6 4 4 3 2 2 2
48332023-09-22T23:25:49.483Z29/30 Write block 301 len 36864 data: 4 4 4 4 3 3 3 3 2
48342023-09-22T23:25:49.486Z30/30 Read block 19 len 16384
48352023-09-22T23:25:49.488Z----------------------------------------------------------------
48362023-09-22T23:25:49.491Z Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30
48372023-09-22T23:25:49.494ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
48382023-09-22T23:25:49.497Z 1 Acked 1000 Write 9 Done Done Done false
48392023-09-22T23:25:49.500Z 2 Acked 1001 Read 6 Done Done Done false
48402023-09-22T23:25:49.502Z 3 Acked 1002 Write 7 Done Done Done false
48412023-09-22T23:25:49.505Z 4 Acked 1003 Write 10 Done Sent Done false
48422023-09-22T23:25:49.508Z 5 Acked 1004 Read 10 Done Sent Done false
48432023-09-22T23:25:49.511Z 6 Acked 1005 Read 3 Done Sent Done false
48442023-09-22T23:25:49.514Z 7 Acked 1006 Write 9 Done Sent Done false
48452023-09-22T23:25:49.517Z 8 Acked 1007 Read 2 Done Sent Done false
48462023-09-22T23:25:49.520Z 9 Acked 1008 Write 6 Done Sent Done false
48472023-09-22T23:25:49.522Z 10 Acked 1009 Read 2 Done Sent Done false
48482023-09-22T23:25:49.525Z 11 Acked 1010 Flush 0 Done Sent Done false
48492023-09-22T23:25:49.528Z 12 Acked 1011 Write 7 Done Sent Done false
48502023-09-22T23:25:49.531Z 13 Acked 1012 Read 9 Done Sent Done false
48512023-09-22T23:25:49.534Z 14 Acked 1013 Write 6 Done Sent Done false
48522023-09-22T23:25:49.537Z 15 Acked 1014 Read 2 Done Sent Done false
48532023-09-22T23:25:49.540Z 16 Acked 1015 Read 3 Done Sent Done false
48542023-09-22T23:25:49.543Z 17 Acked 1016 Write 1 Done Sent Done false
48552023-09-22T23:25:49.546Z 18 Acked 1017 Write 2 Done Sent Done false
48562023-09-22T23:25:49.550Z 19 Acked 1018 Flush 0 Done Sent Done false
48572023-09-22T23:25:49.554Z 20 Acked 1019 Write 4 Done Sent Done false
48582023-09-22T23:25:49.557Z 21 Acked 1020 Write 8 Done Sent Done false
48592023-09-22T23:25:49.560Z 22 Acked 1021 Read 1 Done Sent Done false
48602023-09-22T23:25:49.564Z 23 Acked 1022 Write 4 Done Sent Done false
48612023-09-22T23:25:49.566Z 24 Acked 1023 Write 3 Done Sent Done false
48622023-09-22T23:25:49.569Z 25 Acked 1024 Write 4 Done Sent Done false
48632023-09-22T23:25:49.572Z 26 Acked 1025 Write 4 Done Sent Done false
48642023-09-22T23:25:49.575Z 27 Acked 1026 Read 10 Done Sent Done false
48652023-09-22T23:25:49.578Z 28 Acked 1027 Write 7 Done Sent Done false
48662023-09-22T23:25:49.581Z 29 Acked 1028 Write 9 Done Sent Sent false
48672023-09-22T23:25:49.584Z 30 Acked 1029 Read 4 Done Sent Sent false
48682023-09-22T23:25:49.586Z STATES DS:0 DS:1 DS:2 TOTAL
48692023-09-22T23:25:49.589Z New 0 0 0 0
48702023-09-22T23:25:49.592Z Sent 0 27 2 29
48712023-09-22T23:25:49.595Z Done 30 3 28 61
48722023-09-22T23:25:49.598Z Skipped 0 0 0 0
48732023-09-22T23:25:49.601Z Error 0 0 0 0
48742023-09-22T23:25:49.604ZLast Flush: 1018 0 1018
48752023-09-22T23:25:49.607ZDownstairs last five completed:
48762023-09-22T23:25:49.610ZUpstairs last five completed: 30 29 28 27 26 25
48772023-09-22T23:25:49.612ZWrote out file "/tmp/test_repair_verify.data"
48782023-09-22T23:25:49.616Z
48792023-09-22T23:25:49.618ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
48802023-09-22T23:25:49.621Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.425881548Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1304}
48812023-09-22T23:25:49.624Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.426219317Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1304}
48822023-09-22T23:25:49.627Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.426242442Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1304}
48832023-09-22T23:25:49.631Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.457436785Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1304}
48842023-09-22T23:25:49.633Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.457492832Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1304}
48852023-09-22T23:25:49.636Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.457532561Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1304}
48862023-09-22T23:25:49.639Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.49064571Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1304}
48872023-09-22T23:25:49.642Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.490701739Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1304}
48882023-09-22T23:25:49.644Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:49.49074113Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1304}
48892023-09-22T23:25:49.647ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
48902023-09-22T23:25:49.651Z 0 000-019 31 31 31 45 45 45 F F F
48912023-09-22T23:25:49.653Z 1 020-039 24 24 24 34 34 34 F F F
48922023-09-22T23:25:49.656Z 2 040-059 35 35 35 52 52 52 F F F
48932023-09-22T23:25:49.659Z 3 060-079 35 35 35 52 52 52 F F F
48942023-09-22T23:25:49.662Z 4 080-099 28 28 28 42 42 42 T F T <---
48952023-09-22T23:25:49.665Z 5 100-119 27 27 27 41 41 41 T F T <---
48962023-09-22T23:25:49.668Z 6 120-139 32 32 32 46 46 46 T F T <---
48972023-09-22T23:25:49.671Z 7 140-159 36 34 36 54 50 54 F F F <---
48982023-09-22T23:25:49.674Z 8 160-179 36 31 36 54 45 54 F F F <---
48992023-09-22T23:25:49.678Z 9 180-199 30 30 30 44 44 44 F F F
49002023-09-22T23:25:49.681Z 10 200-219 33 33 33 48 48 48 T F T <---
49012023-09-22T23:25:49.684Z 11 220-239 35 35 35 52 52 52 F F F
49022023-09-22T23:25:49.687Z 12 240-259 35 35 35 52 52 52 F F F
49032023-09-22T23:25:49.689Z 13 260-279 35 35 35 52 52 52 T F T <---
49042023-09-22T23:25:49.692Z 14 280-299 35 35 35 52 52 52 T F T <---
49052023-09-22T23:25:49.695Z 15 300-319 36 35 36 53 52 53 T T T <---
49062023-09-22T23:25:49.698Z 16 320-339 36 35 36 53 52 53 F T F <---
49072023-09-22T23:25:49.700Z 17 340-359 36 35 36 54 52 54 F T F <---
49082023-09-22T23:25:49.703Z 18 360-379 36 34 36 54 50 54 F T F <---
49092023-09-22T23:25:49.705Z 19 380-399 34 34 34 50 50 50 F F F
49102023-09-22T23:25:49.708Z 20 400-419 36 28 36 53 42 53 F F F <---
49112023-09-22T23:25:49.711ZError: Difference in extent metadata found!
49122023-09-22T23:25:49.714Z 21 420-439 36 32 36 53 46 53 F F F <---
49132023-09-22T23:25:49.716Z 22 440-459 34 34 34 50 50 50 F F F
49142023-09-22T23:25:49.719Z 23 460-479 36 34 36 53 50 53 F F F <---
49152023-09-22T23:25:49.722Z 24 480-499 36 34 36 53 49 53 F F F <---
49162023-09-22T23:25:49.725Z 25 500-519 32 32 32 46 46 46 F F F
49172023-09-22T23:25:49.728Z 26 520-539 34 34 34 49 49 49 F F F
49182023-09-22T23:25:49.731Z 27 540-559 34 34 34 49 49 49 T F T <---
49192023-09-22T23:25:49.733Z 28 560-579 32 32 32 47 47 47 F F F
49202023-09-22T23:25:49.736Z 29 580-599 36 30 36 54 44 54 F F F <---
49212023-09-22T23:25:49.739ZMax gen: 36, Max flush: 54
49222023-09-22T23:25:49.742ZA Difference in extent metadata is expected here
49232023-09-22T23:25:49.745ZOn loop 18
49242023-09-22T23:25:49.748Z
49252023-09-22T23:25:49.751ZVerifying data now
49262023-09-22T23:25:56.339ZNo change after verify
49272023-09-22T23:25:56.342ZLoop: 18 Downstairs dump after verify (and repair):
49282023-09-22T23:25:56.358Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.356252537Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1309}
49292023-09-22T23:25:56.361Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.356663365Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1309}
49302023-09-22T23:25:56.363Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.356688114Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1309}
49312023-09-22T23:25:56.376Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.37526025Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1309}
49322023-09-22T23:25:56.379Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.375313624Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1309}
49332023-09-22T23:25:56.382Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.375334668Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1309}
49342023-09-22T23:25:56.388Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.3862101Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1309}
49352023-09-22T23:25:56.391Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.386262197Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1309}
49362023-09-22T23:25:56.394Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.386300787Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1309}
49372023-09-22T23:25:56.400ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
49382023-09-22T23:25:56.403Z 0 000-019 31 31 31 45 45 45 F F F
49392023-09-22T23:25:56.406Z 1 020-039 24 24 24 34 34 34 F F F
49402023-09-22T23:25:56.409Z 2 040-059 35 35 35 52 52 52 F F F
49412023-09-22T23:25:56.411Z 3 060-079 35 35 35 52 52 52 F F F
49422023-09-22T23:25:56.422Z 4 080-099 37 37 37 55 55 55 F F F
49432023-09-22T23:25:56.425Z 5 100-119 37 37 37 55 55 55 F F F
49442023-09-22T23:25:56.428Z 6 120-139 37 37 37 55 55 55 F F F
49452023-09-22T23:25:56.431Z 7 140-159 36 36 36 54 54 54 F F F
49462023-09-22T23:25:56.434Z 8 160-179 36 36 36 54 54 54 F F F
49472023-09-22T23:25:56.437Z 9 180-199 30 30 30 44 44 44 F F F
49482023-09-22T23:25:56.440Z 10 200-219 37 37 37 55 55 55 F F F
49492023-09-22T23:25:56.443Z 11 220-239 35 35 35 52 52 52 F F F
49502023-09-22T23:25:56.446Z 12 240-259 35 35 35 52 52 52 F F F
49512023-09-22T23:25:56.449Z 13 260-279 37 37 37 55 55 55 F F F
49522023-09-22T23:25:56.452Z 14 280-299 37 37 37 55 55 55 F F F
49532023-09-22T23:25:56.455Z 15 300-319 37 37 37 55 55 55 F F F
49542023-09-22T23:25:56.458Z 16 320-339 36 36 36 53 53 53 F F F
49552023-09-22T23:25:56.461Z 17 340-359 36 36 36 54 54 54 F F F
49562023-09-22T23:25:56.465Z 18 360-379 36 36 36 54 54 54 F F F
49572023-09-22T23:25:56.468Z 19 380-399 34 34 34 50 50 50 F F F
49582023-09-22T23:25:56.471Z 20 400-419 36 36 36 53 53 53 F F F
49592023-09-22T23:25:56.474Z 21 420-439 36 36 36 53 53 53 F F F
49602023-09-22T23:25:56.477Z 22 440-459 34 34 34 50 50 50 F F F
49612023-09-22T23:25:56.480Z 23 460-479 36 36 36 53 53 53 F F F
49622023-09-22T23:25:56.484Z 24 480-499 36 36 36 53 53 53 F F F
49632023-09-22T23:25:56.487Z 25 500-519 32 32 32 46 46 46 F F F
49642023-09-22T23:25:56.490Z 26 520-539 34 34 34 49 49 49 F F F
49652023-09-22T23:25:56.493Z 27 540-559 37 37 37 55 55 55 F F F
49662023-09-22T23:25:56.496Z 28 560-579 32 32 32 47 47 47 F F F
49672023-09-22T23:25:56.499Z 29 580-599 36 36 36 54 54 54 F F F
49682023-09-22T23:25:56.502ZMax gen: 37, Max flush: 55
49692023-09-22T23:25:56.505Z
49702023-09-22T23:25:56.508ZBegin loop 19
49712023-09-22T23:25:56.510ZDownstairs to restart: 0
49722023-09-22T23:25:56.513Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.434803588Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
49732023-09-22T23:25:56.516Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.435185613Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
49742023-09-22T23:25:56.519Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.435240341Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
49752023-09-22T23:25:56.522Z{"msg":"Crucible stats registered with UUID: ae774326-859f-4795-8ac5-7dd6d21d1a4d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.435284561Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
49762023-09-22T23:25:56.525Z{"msg":"Crucible ae774326-859f-4795-8ac5-7dd6d21d1a4d has session id: f07afddd-8886-4b53-9542-63c728f706c9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.435344077Z","hostnameCrucible runtime is spawned
49772023-09-22T23:25:56.528Z":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
49782023-09-22T23:25:56.531ZThe guest has requested activation with gen:38
49792023-09-22T23:25:56.535Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.43559831Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311,"task":"up_listen"}
49802023-09-22T23:25:56.538Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.435656475Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
49812023-09-22T23:25:56.541Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.435698315Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
49822023-09-22T23:25:56.544Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.435737532Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311,"looper":"2"}
49832023-09-22T23:25:56.547Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.436036161Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311,"looper":"0"}
49842023-09-22T23:25:56.550Z{"msg":"[2] ae774326-859f-4795-8ac5-7dd6d21d1a4d looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.436100438Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311,"looper":"2"}
49852023-09-22T23:25:56.553Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.436174264Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311,"looper":"1"}
49862023-09-22T23:25:56.556Z{"msg":"[1] ae774326-859f-4795-8ac5-7dd6d21d1a4d looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.436398364Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311,"looper":"1"}
49872023-09-22T23:25:56.560Z{"msg":"ae774326-859f-4795-8ac5-7dd6d21d1a4d active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.436731033Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
49882023-09-22T23:25:56.563Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.436783052Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
49892023-09-22T23:25:56.566Z{"msg":"Set desired generation to :38","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.436824659Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
49902023-09-22T23:25:56.569Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.437051172Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
49912023-09-22T23:25:56.572Z{"msg":"[2] ae774326-859f-4795-8ac5-7dd6d21d1a4d (03b6a7cb-4b8b-4d07-9c9d-4b60d54d3e21) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.437435597Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
49922023-09-22T23:25:56.575Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.437486454Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
49932023-09-22T23:25:56.578Z{"msg":"[2] client is_active_req TRUE, promote! session 03b6a7cb-4b8b-4d07-9c9d-4b60d54d3e21","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.437533247Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
49942023-09-22T23:25:56.581Z{"msg":"[1] ae774326-859f-4795-8ac5-7dd6d21d1a4d (03b6a7cb-4b8b-4d07-9c9d-4b60d54d3e21) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.43759026Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
49952023-09-22T23:25:56.585Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.437641638Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
49962023-09-22T23:25:56.587Z{"msg":"[1] client is_active_req TRUE, promote! session 03b6a7cb-4b8b-4d07-9c9d-4b60d54d3e21","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.437698549Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
49972023-09-22T23:25:56.591Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.438118269Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
49982023-09-22T23:25:56.594Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.438181945Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
49992023-09-22T23:25:56.597Z{"msg":"ae774326-859f-4795-8ac5-7dd6d21d1a4d New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.438285546Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50002023-09-22T23:25:56.600Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.438423049Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50012023-09-22T23:25:56.602Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.438486135Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50022023-09-22T23:25:56.607Z{"msg":"ae774326-859f-4795-8ac5-7dd6d21d1a4d New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.438528097Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50032023-09-22T23:25:56.610Z{"msg":"[2] ae774326-859f-4795-8ac5-7dd6d21d1a4d (03b6a7cb-4b8b-4d07-9c9d-4b60d54d3e21) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.440548591Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50042023-09-22T23:25:56.613Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.440611399Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50052023-09-22T23:25:56.616Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:56.440636581Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50062023-09-22T23:25:56.619Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.440691285Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50072023-09-22T23:25:56.622Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.440812564Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50082023-09-22T23:25:56.625Z{"msg":"ae774326-859f-4795-8ac5-7dd6d21d1a4d New WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.44085659Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50092023-09-22T23:25:56.628Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.440878968Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50102023-09-22T23:25:56.631Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.440916422Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50112023-09-22T23:25:56.634Z{"msg":"[1] ae774326-859f-4795-8ac5-7dd6d21d1a4d (03b6a7cb-4b8b-4d07-9c9d-4b60d54d3e21) New WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.440959215Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50122023-09-22T23:25:56.637Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.440981331Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50132023-09-22T23:25:56.640Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:56.441018251Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50142023-09-22T23:25:56.643Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.441038207Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50152023-09-22T23:25:56.647Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.441055972Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50162023-09-22T23:25:56.658Z{"msg":"ae774326-859f-4795-8ac5-7dd6d21d1a4d New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.441074297Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50172023-09-22T23:25:56.662Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.441091867Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50182023-09-22T23:25:56.665Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:56.44110906Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50192023-09-22T23:25:57.438Z{"msg":"[0] ae774326-859f-4795-8ac5-7dd6d21d1a4d looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.436804337Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311,"looper":"0"}
50202023-09-22T23:25:57.441Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.436895431Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50212023-09-22T23:25:57.444Z{"msg":"[0] ae774326-859f-4795-8ac5-7dd6d21d1a4d (03b6a7cb-4b8b-4d07-9c9d-4b60d54d3e21) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.43803266Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50222023-09-22T23:25:57.447Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.438095742Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50232023-09-22T23:25:57.450Z{"msg":"[0] client is_active_req TRUE, promote! session 03b6a7cb-4b8b-4d07-9c9d-4b60d54d3e21","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.438156803Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50242023-09-22T23:25:57.453Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.438729381Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50252023-09-22T23:25:57.456Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.438796179Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50262023-09-22T23:25:57.459Z{"msg":"ae774326-859f-4795-8ac5-7dd6d21d1a4d WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.438841169Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50272023-09-22T23:25:57.462Z{"msg":"[0] ae774326-859f-4795-8ac5-7dd6d21d1a4d (03b6a7cb-4b8b-4d07-9c9d-4b60d54d3e21) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.441160253Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50282023-09-22T23:25:57.465Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.441220803Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50292023-09-22T23:25:57.468Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:25:57.44124377Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50302023-09-22T23:25:57.471Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.441282509Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50312023-09-22T23:25:57.479Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.441322034Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50322023-09-22T23:25:57.485Z{"msg":"ae774326-859f-4795-8ac5-7dd6d21d1a4d WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.441361209Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50332023-09-22T23:25:57.488Z{"msg":"[0]R flush_numbers[0..12]: [45, 34, 52, 52, 55, 55, 55, 54, 54, 44, 55, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.441400632Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50342023-09-22T23:25:57.491Z{"msg":"[0]R generation[0..12]: [31, 24, 35, 35, 37, 37, 37, 36, 36, 30, 37, 35]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.441440073Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50352023-09-22T23:25:57.494Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.441482379Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50362023-09-22T23:25:57.497Z{"msg":"[1]R flush_numbers[0..12]: [45, 34, 52, 52, 55, 55, 55, 54, 54, 44, 55, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.441521913Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50372023-09-22T23:25:57.500Z{"msg":"[1]R generation[0..12]: [31, 24, 35, 35, 37, 37, 37, 36, 36, 30, 37, 35]"The guest has finished waiting for activation with:38
50382023-09-22T23:25:57.503Z,"v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO
50392023-09-22T23:25:57.506Z30,"time":"2023-09-22T23:25:57.441638519Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50402023-09-22T23:25:57.509Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.441680125Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50412023-09-22T23:25:57.513Z{"msg":"[2]R flush_numbers[0..12]: [45, 34, 52, 52, 55, 55, 55, 54, 54, 44, 55, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.441782894Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50422023-09-22T23:25:57.516Z{"msg":"[2]R generation[0..12]: [31, 24, 35, 35, 37, 37, 37, 36, 36, 30, 37, 35]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.44185289Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50432023-09-22T23:25:57.519Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.44193035Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50442023-09-22T23:25:57.522Z{"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.441950652Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50452023-09-22T23:25:57.525Z{"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.442003571Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50462023-09-22T23:25:57.529Z{"msg":"Next flush: 56","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.44206114Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50472023-09-22T23:25:57.532Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.442084946Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50482023-09-22T23:25:57.535Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.442122235Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50492023-09-22T23:25:57.538Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.442142135Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50502023-09-22T23:25:57.541Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.44220074Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50512023-09-22T23:25:57.545Z{"msg":"ae774326-859f-4795-8ac5-7dd6d21d1a4d is now active with session: 03b6a7cb-4b8b-4d07-9c9d-4b60d54d3e21","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.442246123Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50522023-09-22T23:25:57.548Z{"msg":"ae774326-859f-4795-8ac5-7dd6d21d1a4d Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.442293198Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50532023-09-22T23:25:57.551Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.442342849Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50542023-09-22T23:25:57.554Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.442393801Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50552023-09-22T23:25:57.557Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.442456732Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50562023-09-22T23:25:57.560Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.442499004Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50572023-09-22T23:25:57.563Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.442553315Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50582023-09-22T23:25:57.566ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
50592023-09-22T23:25:57.569Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.442609756Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50602023-09-22T23:25:57.573Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.442649844Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50612023-09-22T23:25:57.576Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.442688088Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50622023-09-22T23:25:57.579Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.44272592Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50632023-09-22T23:25:57.582Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.442764064Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50642023-09-22T23:25:57.585Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.442801667Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1311}
50652023-09-22T23:25:57.588ZLoading write count information from file "/tmp/test_repair_verify.data"
50662023-09-22T23:25:57.591ZRun Repair workload
50672023-09-22T23:25:57.594Z01/30 Write block 132 len 40960 data: 7 5 5 4 5 5 4 4 5 5
50682023-09-22T23:25:57.597Z02/30 Flush
50692023-09-22T23:25:57.600Z03/30 Write block 342 len 20480 data: 3 3 4 5 6
50702023-09-22T23:25:57.603Z04/30 Read block 560 len 32768
50712023-09-22T23:25:57.610Z05/30 Read block 283 len 36864
50722023-09-22T23:25:57.613Z06/30 Read block 379 len 20480
50732023-09-22T23:25:57.616Z07/30 Read block 35 len 8192
50742023-09-22T23:25:57.619Z08/30 Flush
50752023-09-22T23:25:57.622Z09/30 Write block 527 len 12288 data: 3 3 3
50762023-09-22T23:25:57.625Z10/30 Write block 493 len 16384 data: 5 4 4 4
50772023-09-22T23:25:57.627Z11/30 Read block 141 len 8192
50782023-09-22T23:25:57.630Z12/30 Write block 525 len 8192 data: 2 2
50792023-09-22T23:25:57.633Z13/30 Read block 13 len 20480
50802023-09-22T23:25:57.636Z14/30 Read block 168 len 8192
50812023-09-22T23:25:57.639Z15/30 Write block 10 len 40960 data: 3 3 3 4 5 5 4 4 4 4
50822023-09-22T23:25:57.642Z16/30 Write block 449 len 16384 data: 5 5 4 4
50832023-09-22T23:25:57.645Z17/30 Write block 525 len 20480 data: 3 3 4 4 4
50842023-09-22T23:25:57.648Z18/30 Read block 355 len 4096
50852023-09-22T23:25:57.651Z19/30 Read block 384 len 40960
50862023-09-22T23:25:57.654Z20/30 Write block 442 len 16384 data: 3 5 6 6
50872023-09-22T23:25:57.657Z21/30 Read block 230 len 24576
50882023-09-22T23:25:57.660Z22/30 Write block 170 len 40960 data: 5 5 5 4 4 4 4 3 3 3
50892023-09-22T23:25:57.663Z23/30 Read block 192 len 16384
50902023-09-22T23:25:57.666Z24/30 Write block 490 len 32768 data: 6 6 5 6 5 5 5 5
50912023-09-22T23:25:57.669Z25/30 Write block 46 len 12288 data: 5 6 6
50922023-09-22T23:25:57.672Z26/30 Read block 594 len 24576
50932023-09-22T23:25:57.675Z27/30 Write block 428 len 8192 data: 5 5
50942023-09-22T23:25:57.678Z28/30 Read block 373 len 20480
50952023-09-22T23:25:57.681Z29/30 Read block 581 len 8192
50962023-09-22T23:25:57.684Z30/30 Read block 533 len 16384
50972023-09-22T23:25:57.687Z----------------------------------------------------------------
50982023-09-22T23:25:57.690Z Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:28
50992023-09-22T23:25:57.693ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
51002023-09-22T23:25:57.696Z 3 Acked 1002 Write 5 Sent Done Done false
51012023-09-22T23:25:57.699Z 4 Acked 1003 Read 8 Sent Done Done false
51022023-09-22T23:25:57.702Z 5 Acked 1004 Read 9 Sent Done Done false
51032023-09-22T23:25:57.705Z 6 Acked 1005 Read 5 Sent Done Done false
51042023-09-22T23:25:57.708Z 7 Acked 1006 Read 2 Sent Done Done false
51052023-09-22T23:25:57.711Z 8 Acked 1007 Flush 0 Sent Done Done false
51062023-09-22T23:25:57.714Z 9 Acked 1008 Write 3 Sent Done Done false
51072023-09-22T23:25:57.717Z 10 Acked 1009 Write 4 Sent Done Done false
51082023-09-22T23:25:57.720Z 11 Acked 1010 Read 2 Sent Done Done false
51092023-09-22T23:25:57.722Z 12 Acked 1011 Write 2 Sent Done Done false
51102023-09-22T23:25:57.725Z 13 Acked 1012 Read 5 Sent Done Done false
51112023-09-22T23:25:57.728Z 14 Acked 1013 Read 2 Sent Done Done false
51122023-09-22T23:25:57.731Z 15 Acked 1014 Write 10 Sent Done Done false
51132023-09-22T23:25:57.735Z 16 Acked 1015 Write 4 Sent Done Done false
51142023-09-22T23:25:57.738Z 17 Acked 1016 Write 5 Sent Done Done false
51152023-09-22T23:25:57.741Z 18 Acked 1017 Read 1 Sent Done Done false
51162023-09-22T23:25:57.744Z 19 Acked 1018 Read 10 Sent Done Done false
51172023-09-22T23:25:57.747Z 20 Acked 1019 Write 4 Sent Done Done false
51182023-09-22T23:25:57.749Z 21 Acked 1020 Read 6 Sent Done Done false
51192023-09-22T23:25:57.752Z 22 Acked 1021 Write 10 Sent Done Done false
51202023-09-22T23:25:57.755Z 23 Acked 1022 Read 4 Sent Done Done false
51212023-09-22T23:25:57.758Z 24 Acked 1023 Write 8 Sent Done Done false
51222023-09-22T23:25:57.761Z 25 Acked 1024 Write 3 Sent Done Done false
51232023-09-22T23:25:57.764Z 26 Acked 1025 Read 6 Sent Done Done false
51242023-09-22T23:25:57.767Z 27 Acked 1026 Write 2 Sent Done Done false
51252023-09-22T23:25:57.770Z 28 Acked 1027 Read 5 Sent Done Done false
51262023-09-22T23:25:57.773Z 29 Acked 1028 Read 2 Sent Done Done false
51272023-09-22T23:25:57.775Z 30 Acked 1029 Read 4 Sent Sent Done false
51282023-09-22T23:25:57.778Z STATES DS:0 DS:1 DS:2 TOTAL
51292023-09-22T23:25:57.781Z New 0 0 0 0
51302023-09-22T23:25:57.784Z Sent 28 1 0 29
51312023-09-22T23:25:57.788Z Done 0 27 28 55
51322023-09-22T23:25:57.791Z Skipped 0 0 0 0
51332023-09-22T23:25:57.794Z Error 0 0 0 0
51342023-09-22T23:25:57.797ZLast Flush: 1001 1007 1007
51352023-09-22T23:25:57.799ZDownstairs last five completed: 1001 1000
51362023-09-22T23:25:57.802ZUpstairs last five completed: 30 29 28 27 26 25
51372023-09-22T23:25:57.805ZWrote out file "/tmp/test_repair_verify.data"
51382023-09-22T23:25:57.808Z
51392023-09-22T23:25:57.811ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
51402023-09-22T23:25:57.814Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.571093608Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1312}
51412023-09-22T23:25:57.817Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.571360435Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1312}
51422023-09-22T23:25:57.819Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.571383126Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1312}
51432023-09-22T23:25:57.822Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.599005631Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1312}
51442023-09-22T23:25:57.825Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.599040122Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1312}
51452023-09-22T23:25:57.828Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.599059114Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1312}
51462023-09-22T23:25:57.831Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.621192835Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1312}
51472023-09-22T23:25:57.834Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.621226905Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1312}
51482023-09-22T23:25:57.837ZError: Difference in extent metadata found!
51492023-09-22T23:25:57.840Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:25:57.621246177Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1312}
51502023-09-22T23:25:57.843ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51512023-09-22T23:25:57.846Z 0 000-019 31 31 31 45 45 45 F T T <---
51522023-09-22T23:25:57.849Z 1 020-039 24 24 24 34 34 34 F F F
51532023-09-22T23:25:57.852Z 2 040-059 35 35 35 52 52 52 F T T <---
51542023-09-22T23:25:57.854Z 3 060-079 35 35 35 52 52 52 F F F
51552023-09-22T23:25:57.857Z 4 080-099 37 37 37 55 55 55 F F F
51562023-09-22T23:25:57.860Z 5 100-119 37 37 37 55 55 55 F F F
51572023-09-22T23:25:57.864Z 6 120-139 38 38 38 56 56 56 F F F
51582023-09-22T23:25:57.867Z 7 140-159 38 38 38 56 56 56 F F F
51592023-09-22T23:25:57.870Z 8 160-179 36 36 36 54 54 54 F T T <---
51602023-09-22T23:25:57.874Z 9 180-199 30 30 30 44 44 44 F F F
51612023-09-22T23:25:57.876Z 10 200-219 37 37 37 55 55 55 F F F
51622023-09-22T23:25:57.880Z 11 220-239 35 35 35 52 52 52 F F F
51632023-09-22T23:25:57.882Z 12 240-259 35 35 35 52 52 52 F F F
51642023-09-22T23:25:57.885Z 13 260-279 37 37 37 55 55 55 F F F
51652023-09-22T23:25:57.888Z 14 280-299 37 37 37 55 55 55 F F F
51662023-09-22T23:25:57.891Z 15 300-319 37 37 37 55 55 55 F F F
51672023-09-22T23:25:57.894Z 16 320-339 36 36 36 53 53 53 F F F
51682023-09-22T23:25:57.897Z 17 340-359 36 38 38 54 57 57 F F F <---
51692023-09-22T23:25:57.900Z 18 360-379 36 36 36 54 54 54 F F F
51702023-09-22T23:25:57.903Z 19 380-399 34 34 34 50 50 50 F F F
51712023-09-22T23:25:57.905Z 20 400-419 36 36 36 53 53 53 F F F
51722023-09-22T23:25:57.908Z 21 420-439 36 36 36 53 53 53 F T T <---
51732023-09-22T23:25:57.911Z 22 440-459 34 34 34 50 50 50 F T T <---
51742023-09-22T23:25:57.914Z 23 460-479 36 36 36 53 53 53 F F F
51752023-09-22T23:25:57.917Z 24 480-499 36 36 36 53 53 53 F T T <---
51762023-09-22T23:25:57.920Z 25 500-519 32 32 32 46 46 46 F F F
51772023-09-22T23:25:57.923Z 26 520-539 34 34 34 49 49 49 F T T <---
51782023-09-22T23:25:57.926Z 27 540-559 37 37 37 55 55 55 F F F
51792023-09-22T23:25:57.940Z 28 560-579 32 32 32 47 47 47 F F F
51802023-09-22T23:25:57.943Z 29 580-599 36 36 36 54 54 54 F F F
51812023-09-22T23:25:57.946ZMax gen: 38, Max flush: 57
51822023-09-22T23:25:57.950ZA Difference in extent metadata is expected here
51832023-09-22T23:25:57.952ZOn loop 19
51842023-09-22T23:25:57.956Z
51852023-09-22T23:25:57.958ZVerifying data now
51862023-09-22T23:26:03.663ZNo change after verify
51872023-09-22T23:26:03.666ZLoop: 19 Downstairs dump after verify (and repair):
51882023-09-22T23:26:03.681Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.679804238Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1317}
51892023-09-22T23:26:03.685Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.680204933Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1317}
51902023-09-22T23:26:03.688Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.680229978Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1317}
51912023-09-22T23:26:03.700Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.699066331Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1317}
51922023-09-22T23:26:03.704Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.699120731Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1317}
51932023-09-22T23:26:03.708Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.699141293Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1317}
51942023-09-22T23:26:03.711Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.709883627Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1317}
51952023-09-22T23:26:03.714Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.709935888Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1317}
51962023-09-22T23:26:03.720Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.709958063Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1317}
51972023-09-22T23:26:03.723ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
51982023-09-22T23:26:03.727Z 0 000-019 39 39 39 58 58 58 F F F
51992023-09-22T23:26:03.731Z 1 020-039 24 24 24 34 34 34 F F F
52002023-09-22T23:26:03.735Z 2 040-059 39 39 39 58 58 58 F F F
52012023-09-22T23:26:03.738Z 3 060-079 35 35 35 52 52 52 F F F
52022023-09-22T23:26:03.741Z 4 080-099 37 37 37 55 55 55 F F F
52032023-09-22T23:26:03.745Z 5 100-119 37 37 37 55 55 55 F F F
52042023-09-22T23:26:03.748Z 6 120-139 38 38 38 56 56 56 F F F
52052023-09-22T23:26:03.753Z 7 140-159 38 38 38 56 56 56 F F F
52062023-09-22T23:26:03.757Z 8 160-179 39 39 39 58 58 58 F F F
52072023-09-22T23:26:03.760Z 9 180-199 30 30 30 44 44 44 F F F
52082023-09-22T23:26:03.765Z 10 200-219 37 37 37 55 55 55 F F F
52092023-09-22T23:26:03.768Z 11 220-239 35 35 35 52 52 52 F F F
52102023-09-22T23:26:03.772Z 12 240-259 35 35 35 52 52 52 F F F
52112023-09-22T23:26:03.775Z 13 260-279 37 37 37 55 55 55 F F F
52122023-09-22T23:26:03.781Z 14 280-299 37 37 37 55 55 55 F F F
52132023-09-22T23:26:03.784Z 15 300-319 37 37 37 55 55 55 F F F
52142023-09-22T23:26:03.787Z 16 320-339 36 36 36 53 53 53 F F F
52152023-09-22T23:26:03.791Z 17 340-359 38 38 38 57 57 57 F F F
52162023-09-22T23:26:03.794Z 18 360-379 36 36 36 54 54 54 F F F
52172023-09-22T23:26:03.797Z 19 380-399 34 34 34 50 50 50 F F F
52182023-09-22T23:26:03.801Z 20 400-419 36 36 36 53 53 53 F F F
52192023-09-22T23:26:03.804Z 21 420-439 39 39 39 58 58 58 F F F
52202023-09-22T23:26:03.809Z 22 440-459 39 39 39 58 58 58 F F F
52212023-09-22T23:26:03.813Z 23 460-479 36 36 36 53 53 53 F F F
52222023-09-22T23:26:03.816Z 24 480-499 39 39 39 58 58 58 F F F
52232023-09-22T23:26:03.820Z 25 500-519 32 32 32 46 46 46 F F F
52242023-09-22T23:26:03.823Z 26 520-539 39 39 39 58 58 58 F F F
52252023-09-22T23:26:03.826Z 27 540-559 37 37 37 55 55 55 F F F
52262023-09-22T23:26:03.830Z 28 560-579 32 32 32 47 47 47 F F F
52272023-09-22T23:26:03.835Z 29 580-599 36 36 36 54 54 54 F F F
52282023-09-22T23:26:03.838ZMax gen: 39, Max flush: 58
52292023-09-22T23:26:03.841Z
52302023-09-22T23:26:03.844ZBegin loop 20
52312023-09-22T23:26:03.848ZDownstairs to restart: 1
52322023-09-22T23:26:03.851Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.759823333Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52332023-09-22T23:26:03.855Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.760188967Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52342023-09-22T23:26:03.860Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.760241877Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52352023-09-22T23:26:03.863Z{"msg":"Crucible stats registered with UUID: 316290bc-2c08-4906-b782-a8405906e96d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.760306555Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52362023-09-22T23:26:03.866Z{"msg":"Crucible 316290bc-2c08-4906-b782-a8405906e96d has session id: 6454481c-057a-4301-9c6f-b4ba65cf5c99","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.760370838Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52372023-09-22T23:26:03.870ZCrucible runtime is spawned
52382023-09-22T23:26:03.875ZThe guest has requested activation with gen:40
52392023-09-22T23:26:03.878Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.760653978Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319,"task":"up_listen"}
52402023-09-22T23:26:03.881Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.760767279Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52412023-09-22T23:26:03.885Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.760841457Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52422023-09-22T23:26:03.888Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.760910114Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319,"looper":"2"}
52432023-09-22T23:26:03.892Z{"msg":"[2] 316290bc-2c08-4906-b782-a8405906e96d looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.761078062Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319,"looper":"2"}
52442023-09-22T23:26:03.895Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.761188082Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319,"looper":"1"}
52452023-09-22T23:26:03.898Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.761320204Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319,"looper":"0"}
52462023-09-22T23:26:03.902Z{"msg":"[0] 316290bc-2c08-4906-b782-a8405906e96d looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.761556335Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319,"looper":"0"}
52472023-09-22T23:26:03.906Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.761683446Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52482023-09-22T23:26:03.909Z{"msg":"316290bc-2c08-4906-b782-a8405906e96d active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.761776791Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52492023-09-22T23:26:03.915Z{"msg":"Set desired generation to :40","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.761887853Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52502023-09-22T23:26:03.919Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.762073991Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52512023-09-22T23:26:03.923Z{"msg":"[2] 316290bc-2c08-4906-b782-a8405906e96d (f83ea62b-50fe-4ac0-9e29-e4dd9e39d3d4) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.7623891Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52522023-09-22T23:26:03.926Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.762470466Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52532023-09-22T23:26:03.930Z{"msg":"[2] client is_active_req TRUE, promote! session f83ea62b-50fe-4ac0-9e29-e4dd9e39d3d4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.762548156Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52542023-09-22T23:26:03.933Z{"msg":"[0] 316290bc-2c08-4906-b782-a8405906e96d (f83ea62b-50fe-4ac0-9e29-e4dd9e39d3d4) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.76263176Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52552023-09-22T23:26:03.936Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.762701373Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52562023-09-22T23:26:03.944Z{"msg":"[0] client is_active_req TRUE, promote! session f83ea62b-50fe-4ac0-9e29-e4dd9e39d3d4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.762746525Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52572023-09-22T23:26:03.949Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.763150597Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52582023-09-22T23:26:03.953Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.763220639Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52592023-09-22T23:26:03.959Z{"msg":"316290bc-2c08-4906-b782-a8405906e96d WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.763266363Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52602023-09-22T23:26:03.963Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.763415773Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52612023-09-22T23:26:03.967Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.763478076Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52622023-09-22T23:26:03.970Z{"msg":"316290bc-2c08-4906-b782-a8405906e96d WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.763519652Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52632023-09-22T23:26:03.974Z{"msg":"[2] 316290bc-2c08-4906-b782-a8405906e96d (f83ea62b-50fe-4ac0-9e29-e4dd9e39d3d4) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.765569783Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52642023-09-22T23:26:03.978Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.765647899Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52652023-09-22T23:26:03.981Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:03.765697424Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52662023-09-22T23:26:03.986Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.765737318Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52672023-09-22T23:26:03.989Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.76577698Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52682023-09-22T23:26:03.993Z{"msg":"316290bc-2c08-4906-b782-a8405906e96d WaitActive New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.765816625Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52692023-09-22T23:26:03.999Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.765855462Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52702023-09-22T23:26:04.005Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.76589373Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52712023-09-22T23:26:04.008Z{"msg":"[0] 316290bc-2c08-4906-b782-a8405906e96d (f83ea62b-50fe-4ac0-9e29-e4dd9e39d3d4) WaitActive New WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.766019658Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52722023-09-22T23:26:04.011Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.766069846Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52732023-09-22T23:26:04.015Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:03.766110229Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52742023-09-22T23:26:04.018Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.766148809Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52752023-09-22T23:26:04.022Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.766169192Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52762023-09-22T23:26:04.025Z{"msg":"316290bc-2c08-4906-b782-a8405906e96d WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.766187464Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52772023-09-22T23:26:04.028Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.7662216Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52782023-09-22T23:26:04.032Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:03.766241143Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52792023-09-22T23:26:04.764Z{"msg":"[1] 316290bc-2c08-4906-b782-a8405906e96d looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.762286Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319,"looper":"1"}
52802023-09-22T23:26:04.767Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.762369034Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52812023-09-22T23:26:04.770Z{"msg":"[1] 316290bc-2c08-4906-b782-a8405906e96d (f83ea62b-50fe-4ac0-9e29-e4dd9e39d3d4) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.763528271Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52822023-09-22T23:26:04.773Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.763583484Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52832023-09-22T23:26:04.777Z{"msg":"[1] client is_active_req TRUE, promote! session f83ea62b-50fe-4ac0-9e29-e4dd9e39d3d4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.763626591Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52842023-09-22T23:26:04.779Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.7641928Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52852023-09-22T23:26:04.782Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.764250949Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52862023-09-22T23:26:04.786Z{"msg":"316290bc-2c08-4906-b782-a8405906e96d WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.764294383Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52872023-09-22T23:26:04.789Z{"msg":"[1] 316290bc-2c08-4906-b782-a8405906e96d (f83ea62b-50fe-4ac0-9e29-e4dd9e39d3d4) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.766584133Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52882023-09-22T23:26:04.792Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.766619549Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52892023-09-22T23:26:04.795Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:04.766675391Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52902023-09-22T23:26:04.798Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.766715504Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52912023-09-22T23:26:04.801Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.766802727Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52922023-09-22T23:26:04.809Z{"msg":"316290bc-2c08-4906-b782-a8405906e96d WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.766824379Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52932023-09-22T23:26:04.812Z{"msg":"[0]R flush_numbers[0..12]: [58, 34, 58, 52, 55, 55, 56, 56, 58, 44, 55, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.766849094Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52942023-09-22T23:26:04.815Z{"msg":"[0]R generation[0..12]: [39, 24, 39, 35, 37, 37, 38, 38, 39, 30, 37, 35]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.766893091Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52952023-09-22T23:26:04.818Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.766939076Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52962023-09-22T23:26:04.821Z{"msg":"[1]R flush_numbers[0..12]: [58, 34, 58, 52, 55, 55, 56, 56, 58, 44, 55, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.76699083Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52972023-09-22T23:26:04.824Z{"msg":"[1]R generation[0..12]: [39, 24, 39, 35, 37, 37, 38, 38, 39, 30, 37, 35]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.767011757Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
52982023-09-22T23:26:04.828Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:40
52992023-09-22T23:26:04.831ZWait for a query_work_queue command to finish before sending IO
53002023-09-22T23:26:04.835Z,"time":"2023-09-22T23:26:04.767066306Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
53012023-09-22T23:26:04.838Z{"msg":"[2]R flush_numbers[0..12]: [58, 34, 58, 52, 55, 55, 56, 56, 58, 44, 55, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.767238373Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
53022023-09-22T23:26:04.841Z{"msg":"[2]R generation[0..12]: [39, 24, 39, 35, 37, 37, 38, 38, 39, 30, 37, 35]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.767315629Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
53032023-09-22T23:26:04.844Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.767363061Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
53042023-09-22T23:26:04.847Z{"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.767401576Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
53052023-09-22T23:26:04.851Z{"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.767449326Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
53062023-09-22T23:26:04.854Z{"msg":"Next flush: 59","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.767497896Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
53072023-09-22T23:26:04.857Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.767544469Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
53082023-09-22T23:26:04.860Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.767595168Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
53092023-09-22T23:26:04.863Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.767642986Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
53102023-09-22T23:26:04.867Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.767687493Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
53112023-09-22T23:26:04.870Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
53122023-09-22T23:26:04.873Z316290bc-2c08-4906-b782-a8405906e96d is now active with session: f83ea62b-50fe-4ac0-9e29-e4dd9e39d3d4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.767731578Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
53132023-09-22T23:26:04.876Z{"msg":"316290bc-2c08-4906-b782-a8405906e96d Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.767771513Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
53142023-09-22T23:26:04.879Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.767810063Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
53152023-09-22T23:26:04.882Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.767847943Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
53162023-09-22T23:26:04.885Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.767886097Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
53172023-09-22T23:26:04.889Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.767924167Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
53182023-09-22T23:26:04.892Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.767961724Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
53192023-09-22T23:26:04.895Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.767999822Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
53202023-09-22T23:26:04.898Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.768037502Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
53212023-09-22T23:26:04.901Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.768075262Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
53222023-09-22T23:26:04.904Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.768112905Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
53232023-09-22T23:26:04.907Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.76815097Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
53242023-09-22T23:26:04.911Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.768188702Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1319}
53252023-09-22T23:26:04.914ZLoading write count information from file "/tmp/test_repair_verify.data"
53262023-09-22T23:26:04.917ZRun Repair workload
53272023-09-22T23:26:04.920Z01/30 Flush
53282023-09-22T23:26:04.923Z02/30 Write block 131 len 24576 data: 4 8 6 6 5 6
53292023-09-22T23:26:04.926Z03/30 Write block 402 len 40960 data: 6 5 5 5 5 5 5 6 5 6
53302023-09-22T23:26:04.929Z04/30 Read block 445 len 4096
53312023-09-22T23:26:04.932Z05/30 Read block 398 len 28672
53322023-09-22T23:26:04.935Z06/30 Write block 507 len 12288 data: 4 3 5
53332023-09-22T23:26:04.938Z07/30 Read block 61 len 12288
53342023-09-22T23:26:04.941Z08/30 Write block 263 len 40960 data: 7 7 7 7 8 6 7 7 7 7
53352023-09-22T23:26:04.944Z09/30 Flush
53362023-09-22T23:26:04.948Z10/30 Write block 487 len 12288 data: 4 6 7
53372023-09-22T23:26:04.951Z11/30 Write block 448 len 40960 data: 5 6 6 5 5 4 5 5 4 5
53382023-09-22T23:26:04.954Z12/30 Read block 584 len 24576
53392023-09-22T23:26:04.957Z13/30 Write block 414 len 16384 data: 4 3 3 2
53402023-09-22T23:26:04.960Z14/30 Write block 444 len 12288 data: 7 7 6
53412023-09-22T23:26:04.963Z15/30 Write block 194 len 12288 data: 5 5 4
53422023-09-22T23:26:04.966Z16/30 Write block 370 len 36864 data: 2 2 2 2 3 3 4 5 6
53432023-09-22T23:26:04.969Z17/30 Write block 258 len 28672 data: 3 3 4 6 7 8 8
53442023-09-22T23:26:04.972Z18/30 Write block 280 len 40960 data: 6 5 6 6 5 5 6 6 5 5
53452023-09-22T23:26:04.975Z19/30 Read block 272 len 4096
53462023-09-22T23:26:04.979Z20/30 Read block 96 len 16384
53472023-09-22T23:26:04.982Z21/30 Flush
53482023-09-22T23:26:04.985Z22/30 Write block 413 len 36864 data: 4 5 4 4 3 2 2 2 2
53492023-09-22T23:26:04.989Z23/30 Write block 1 len 12288 data: 2 3 4
53502023-09-22T23:26:04.992Z24/30 Flush
53512023-09-22T23:26:04.995Z25/30 Write block 451 len 24576 data: 6 6 5 6 6 5
53522023-09-22T23:26:04.998Z26/30 Write block 188 len 40960 data: 3 5 6 5 4 4 6 6 5 4
53532023-09-22T23:26:05.002Z27/30 Write block 558 len 16384 data: 4 4 4 4
53542023-09-22T23:26:05.005Z28/30 Write block 197 len 4096 data: 5
53552023-09-22T23:26:05.008Z29/30 Write block 69 len 24576 data: 10 7 7 6 3 2
53562023-09-22T23:26:05.011Z30/30 Write block 301 len 36864 data: 5 5 5 5 4 4 4 4 3
53572023-09-22T23:26:05.015Z----------------------------------------------------------------
53582023-09-22T23:26:05.018Z Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:29
53592023-09-22T23:26:05.021ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
53602023-09-22T23:26:05.024Z 2 Acked 1001 Write 6 Done Done Done false
53612023-09-22T23:26:05.027Z 3 Acked 1002 Write 10 Done Done Done false
53622023-09-22T23:26:05.031Z 4 Acked 1003 Read 1 Done Done Done false
53632023-09-22T23:26:05.035Z 5 Acked 1004 Read 7 Done Done Done false
53642023-09-22T23:26:05.038Z 6 Acked 1005 Write 3 Done Sent Done false
53652023-09-22T23:26:05.041Z 7 Acked 1006 Read 3 Done Sent Done false
53662023-09-22T23:26:05.044Z 8 Acked 1007 Write 10 Done Sent Done false
53672023-09-22T23:26:05.047Z 9 Acked 1008 Flush 0 Done Sent Done false
53682023-09-22T23:26:05.050Z 10 Acked 1009 Write 3 Done Sent Done false
53692023-09-22T23:26:05.053Z 11 Acked 1010 Write 10 Done Sent Done false
53702023-09-22T23:26:05.056Z 12 Acked 1011 Read 6 Done Sent Done false
53712023-09-22T23:26:05.062Z 13 Acked 1012 Write 4 Done Sent Done false
53722023-09-22T23:26:05.065Z 14 Acked 1013 Write 3 Done Sent Done false
53732023-09-22T23:26:05.068Z 15 Acked 1014 Write 3 Done Sent Done false
53742023-09-22T23:26:05.071Z 16 Acked 1015 Write 9 Done Sent Done false
53752023-09-22T23:26:05.074Z 17 Acked 1016 Write 7 Done Sent Done false
53762023-09-22T23:26:05.077Z 18 Acked 1017 Write 10 Done Sent Done false
53772023-09-22T23:26:05.081Z 19 Acked 1018 Read 1 Done Sent Done false
53782023-09-22T23:26:05.084Z 20 Acked 1019 Read 4 Done Sent Done false
53792023-09-22T23:26:05.087Z 21 Acked 1020 Flush 0 Done Sent Done false
53802023-09-22T23:26:05.090Z 22 Acked 1021 Write 9 Done Sent Done false
53812023-09-22T23:26:05.093Z 23 Acked 1022 Write 3 Done Sent Done false
53822023-09-22T23:26:05.096Z 24 Acked 1023 Flush 0 Done Sent Done false
53832023-09-22T23:26:05.100Z 25 Acked 1024 Write 6 Done Sent Done false
53842023-09-22T23:26:05.103Z 26 Acked 1025 Write 10 Done Sent Done false
53852023-09-22T23:26:05.106Z 27 Acked 1026 Write 4 Sent Sent Sent false
53862023-09-22T23:26:05.109Z 28 Acked 1027 Write 1 Sent Sent Sent false
53872023-09-22T23:26:05.112Z 29 Acked 1028 Write 6 Sent Sent Sent false
53882023-09-22T23:26:05.115Z 30 Acked 1029 Write 9 Sent Sent Sent false
53892023-09-22T23:26:05.118Z STATES DS:0 DS:1 DS:2 TOTAL
53902023-09-22T23:26:05.121Z New 0 0 0 0
53912023-09-22T23:26:05.124Z Sent 4 25 4 33
53922023-09-22T23:26:05.127Z Done 25 4 25 54
53932023-09-22T23:26:05.130Z Skipped 0 0 0 0
53942023-09-22T23:26:05.134Z Error 0 0 0 0
53952023-09-22T23:26:05.137ZLast Flush: 1023 1000 1023
53962023-09-22T23:26:05.140ZDownstairs last five completed: 1000
53972023-09-22T23:26:05.143ZUpstairs last five completed: 30 29 28 27 26 25
53982023-09-22T23:26:05.146ZWrote out file "/tmp/test_repair_verify.data"
53992023-09-22T23:26:05.149Z
54002023-09-22T23:26:05.152ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
54012023-09-22T23:26:05.155Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.891071502Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1320}
54022023-09-22T23:26:05.159Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.891384347Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1320}
54032023-09-22T23:26:05.171Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.891407703Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1320}
54042023-09-22T23:26:05.174Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.917273339Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1320}
54052023-09-22T23:26:05.177Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.917306769Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1320}
54062023-09-22T23:26:05.180ZError: Difference in extent metadata found!
54072023-09-22T23:26:05.183Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.917325914Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1320}
54082023-09-22T23:26:05.187Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.943811373Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1320}
54092023-09-22T23:26:05.190Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.943844362Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1320}
54102023-09-22T23:26:05.193Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:04.943863927Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1320}
54112023-09-22T23:26:05.196ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
54122023-09-22T23:26:05.199Z 0 000-019 40 39 40 62 58 62 F F F <---
54132023-09-22T23:26:05.202Z 1 020-039 24 24 24 34 34 34 F F F
54142023-09-22T23:26:05.205Z 2 040-059 39 39 39 58 58 58 F F F
54152023-09-22T23:26:05.208Z 3 060-079 35 35 35 52 52 52 F F F
54162023-09-22T23:26:05.211Z 4 080-099 37 37 37 55 55 55 F F F
54172023-09-22T23:26:05.214Z 5 100-119 37 37 37 55 55 55 F F F
54182023-09-22T23:26:05.219Z 6 120-139 40 38 40 60 56 60 F T F <---
54192023-09-22T23:26:05.222Z 7 140-159 38 38 38 56 56 56 F F F
54202023-09-22T23:26:05.225Z 8 160-179 39 39 39 58 58 58 F F F
54212023-09-22T23:26:05.228Z 9 180-199 40 30 40 61 44 61 T F T <---
54222023-09-22T23:26:05.232Z 10 200-219 37 37 37 55 55 55 F F F
54232023-09-22T23:26:05.235Z 11 220-239 35 35 35 52 52 52 F F F
54242023-09-22T23:26:05.238Z 12 240-259 40 35 40 61 52 61 F F F <---
54252023-09-22T23:26:05.241Z 13 260-279 40 37 40 61 55 61 F F F <---
54262023-09-22T23:26:05.244Z 14 280-299 40 37 40 61 55 61 F F F <---
54272023-09-22T23:26:05.247Z 15 300-319 37 37 37 55 55 55 F F F
54282023-09-22T23:26:05.250Z 16 320-339 36 36 36 53 53 53 F F F
54292023-09-22T23:26:05.253Z 17 340-359 38 38 38 57 57 57 F F F
54302023-09-22T23:26:05.256Z 18 360-379 40 36 40 61 54 61 F F F <---
54312023-09-22T23:26:05.260Z 19 380-399 34 34 34 50 50 50 F F F
54322023-09-22T23:26:05.263Z 20 400-419 40 36 40 62 53 62 F T F <---
54332023-09-22T23:26:05.265Z 21 420-439 40 39 40 62 58 62 F F F <---
54342023-09-22T23:26:05.269Z 22 440-459 40 39 40 61 58 61 T F T <---
54352023-09-22T23:26:05.272Z 23 460-479 36 36 36 53 53 53 F F F
54362023-09-22T23:26:05.275Z 24 480-499 40 39 40 61 58 61 F F F <---
54372023-09-22T23:26:05.278Z 25 500-519 40 32 40 60 46 60 F F F <---
54382023-09-22T23:26:05.281Z 26 520-539 39 39 39 58 58 58 F F F
54392023-09-22T23:26:05.284Z 27 540-559 37 37 37 55 55 55 T F T <---
54402023-09-22T23:26:05.287Z 28 560-579 32 32 32 47 47 47 T F T <---
54412023-09-22T23:26:05.290Z 29 580-599 36 36 36 54 54 54 F F F
54422023-09-22T23:26:05.293ZMax gen: 40, Max flush: 62
54432023-09-22T23:26:05.296ZA Difference in extent metadata is expected here
54442023-09-22T23:26:05.299ZOn loop 20
54452023-09-22T23:26:05.302Z
54462023-09-22T23:26:05.305ZVerifying data now
54472023-09-22T23:26:11.657ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
54482023-09-22T23:26:11.661Z72c72,74
54492023-09-22T23:26:11.664Z< 9,
54502023-09-22T23:26:11.667Z---
54512023-09-22T23:26:11.670Z> 10,
54522023-09-22T23:26:11.673Z> 7,
54532023-09-22T23:26:11.676Z> 7,
54542023-09-22T23:26:11.679Z74,75c76
54552023-09-22T23:26:11.682Z< 6,
54562023-09-22T23:26:11.685Z< 5,
54572023-09-22T23:26:11.688Z---
54582023-09-22T23:26:11.691Z> 3,
54592023-09-22T23:26:11.694Z77d77
54602023-09-22T23:26:11.697Z< 1,
54612023-09-22T23:26:11.700Z303a304,307
54622023-09-22T23:26:11.703Z> 5,
54632023-09-22T23:26:11.706Z> 5,
54642023-09-22T23:26:11.709Z> 5,
54652023-09-22T23:26:11.713Z> 5,
54662023-09-22T23:26:11.716Z309,312d312
54672023-09-22T23:26:11.719Z< 3,
54682023-09-22T23:26:11.722Z< 3,
54692023-09-22T23:26:11.725Z< 3,
54702023-09-22T23:26:11.731Z< 2,
54712023-09-22T23:26:11.736Zdiff found after verify
54722023-09-22T23:26:11.739ZLoop: 20 Downstairs dump after verify (and repair):
54732023-09-22T23:26:11.742Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.684029055Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1328}
54742023-09-22T23:26:11.746Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.6843668Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1328}
54752023-09-22T23:26:11.749Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.684411745Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1328}
54762023-09-22T23:26:11.752Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.703200761Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1328}
54772023-09-22T23:26:11.755Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.703253593Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1328}
54782023-09-22T23:26:11.758Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.703291984Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1328}
54792023-09-22T23:26:11.761Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.71420514Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1328}
54802023-09-22T23:26:11.764Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.714257119Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1328}
54812023-09-22T23:26:11.769Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.714295127Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1328}
54822023-09-22T23:26:11.772ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
54832023-09-22T23:26:11.775Z 0 000-019 40 40 40 62 62 62 F F F
54842023-09-22T23:26:11.778Z 1 020-039 24 24 24 34 34 34 F F F
54852023-09-22T23:26:11.781Z 2 040-059 39 39 39 58 58 58 F F F
54862023-09-22T23:26:11.785Z 3 060-079 35 35 35 52 52 52 F F F
54872023-09-22T23:26:11.788Z 4 080-099 37 37 37 55 55 55 F F F
54882023-09-22T23:26:11.791Z 5 100-119 37 37 37 55 55 55 F F F
54892023-09-22T23:26:11.794Z 6 120-139 40 40 40 60 60 60 F F F
54902023-09-22T23:26:11.797Z 7 140-159 38 38 38 56 56 56 F F F
54912023-09-22T23:26:11.800Z 8 160-179 39 39 39 58 58 58 F F F
54922023-09-22T23:26:11.803Z 9 180-199 41 41 41 63 63 63 F F F
54932023-09-22T23:26:11.806Z 10 200-219 37 37 37 55 55 55 F F F
54942023-09-22T23:26:11.809Z 11 220-239 35 35 35 52 52 52 F F F
54952023-09-22T23:26:11.812Z 12 240-259 40 40 40 61 61 61 F F F
54962023-09-22T23:26:11.815Z 13 260-279 40 40 40 61 61 61 F F F
54972023-09-22T23:26:11.817Z 14 280-299 40 40 40 61 61 61 F F F
54982023-09-22T23:26:11.820Z 15 300-319 37 37 37 55 55 55 F F F
54992023-09-22T23:26:11.823Z 16 320-339 36 36 36 53 53 53 F F F
55002023-09-22T23:26:11.826Z 17 340-359 38 38 38 57 57 57 F F F
55012023-09-22T23:26:11.829Z 18 360-379 40 40 40 61 61 61 F F F
55022023-09-22T23:26:11.832Z 19 380-399 34 34 34 50 50 50 F F F
55032023-09-22T23:26:11.835Z 20 400-419 40 40 40 62 62 62 F F F
55042023-09-22T23:26:11.838Z 21 420-439 40 40 40 62 62 62 F F F
55052023-09-22T23:26:11.841Z 22 440-459 41 41 41 63 63 63 F F F
55062023-09-22T23:26:11.844Z 23 460-479 36 36 36 53 53 53 F F F
55072023-09-22T23:26:11.847Z 24 480-499 40 40 40 61 61 61 F F F
55082023-09-22T23:26:11.850Z 25 500-519 40 40 40 60 60 60 F F F
55092023-09-22T23:26:11.854Z 26 520-539 39 39 39 58 58 58 F F F
55102023-09-22T23:26:11.857Z 27 540-559 41 41 41 63 63 63 F F F
55112023-09-22T23:26:11.860Z 28 560-579 41 41 41 63 63 63 F F F
55122023-09-22T23:26:11.863Z 29 580-599 36 36 36 54 54 54 F F F
55132023-09-22T23:26:11.866ZMax gen: 41, Max flush: 63
55142023-09-22T23:26:11.869Z
55152023-09-22T23:26:11.872ZBegin loop 21
55162023-09-22T23:26:11.875ZDownstairs to restart: 0
55172023-09-22T23:26:11.878Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.764983438Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55182023-09-22T23:26:11.881Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.765359351Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55192023-09-22T23:26:11.885Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.765420201Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55202023-09-22T23:26:11.888Z{"msg":"Crucible stats registered with UUID: 78dec864-2600-4c71-8d63-dbd740e3526c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.765446145Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55212023-09-22T23:26:11.890Z{"msg":"Crucible 78dec864-2600-4c71-8d63-dbd740e3526c has session id: a942619b-a961-4835-afa2-42acf9f9e9f7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.765500028Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55222023-09-22T23:26:11.894ZCrucible runtime is spawned
55232023-09-22T23:26:11.897ZThe guest has requested activation with gen:42
55242023-09-22T23:26:11.900Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.765836271Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330,"task":"up_listen"}
55252023-09-22T23:26:11.903Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.765894716Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55262023-09-22T23:26:11.905Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.765936823Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55272023-09-22T23:26:11.909Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.765977286Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330,"looper":"2"}
55282023-09-22T23:26:11.912Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.766293194Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330,"looper":"1"}
55292023-09-22T23:26:11.915Z{"msg":"[2] 78dec864-2600-4c71-8d63-dbd740e3526c looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.766368848Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330,"looper":"2"}
55302023-09-22T23:26:11.918Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.766430865Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330,"looper":"0"}
55312023-09-22T23:26:11.921Z{"msg":"[1] 78dec864-2600-4c71-8d63-dbd740e3526c looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.766500729Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330,"looper":"1"}
55322023-09-22T23:26:11.924Z{"msg":"78dec864-2600-4c71-8d63-dbd740e3526c active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.766864312Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55332023-09-22T23:26:11.927Z{"msg":"Set desired generation to :42","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.766938962Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55342023-09-22T23:26:11.931Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.767136611Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55352023-09-22T23:26:11.934Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.767447469Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55362023-09-22T23:26:11.937Z{"msg":"[2] 78dec864-2600-4c71-8d63-dbd740e3526c (8eba43b0-4e73-44df-ad6b-f5657162f6a4) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.767797635Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55372023-09-22T23:26:11.940Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.767860073Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55382023-09-22T23:26:11.943Z{"msg":"[2] client is_active_req TRUE, promote! session 8eba43b0-4e73-44df-ad6b-f5657162f6a4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.767919183Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55392023-09-22T23:26:11.947Z{"msg":"[1] 78dec864-2600-4c71-8d63-dbd740e3526c (8eba43b0-4e73-44df-ad6b-f5657162f6a4) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.767986347Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55402023-09-22T23:26:11.950Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.768052557Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55412023-09-22T23:26:11.953Z{"msg":"[1] client is_active_req TRUE, promote! session 8eba43b0-4e73-44df-ad6b-f5657162f6a4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.768110201Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55422023-09-22T23:26:11.956Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.768484188Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55432023-09-22T23:26:11.959Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.76855847Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55442023-09-22T23:26:11.962Z{"msg":"78dec864-2600-4c71-8d63-dbd740e3526c New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.768614838Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55452023-09-22T23:26:11.965Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.768716991Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55462023-09-22T23:26:11.968Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.768788192Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55472023-09-22T23:26:11.970Z{"msg":"78dec864-2600-4c71-8d63-dbd740e3526c New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.768832982Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55482023-09-22T23:26:11.973Z{"msg":"[2] 78dec864-2600-4c71-8d63-dbd740e3526c (8eba43b0-4e73-44df-ad6b-f5657162f6a4) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.77080671Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55492023-09-22T23:26:11.976Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.770842255Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55502023-09-22T23:26:11.979Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:11.770862235Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55512023-09-22T23:26:11.982Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.770884827Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55522023-09-22T23:26:11.985Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.770977263Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55532023-09-22T23:26:11.988Z{"msg":"78dec864-2600-4c71-8d63-dbd740e3526c New WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.771003954Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55542023-09-22T23:26:11.991Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.771023024Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55552023-09-22T23:26:11.994Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.771040721Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55562023-09-22T23:26:11.997Z{"msg":"[1] 78dec864-2600-4c71-8d63-dbd740e3526c (8eba43b0-4e73-44df-ad6b-f5657162f6a4) New WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.772063571Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55572023-09-22T23:26:12.001Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.772094584Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55582023-09-22T23:26:12.004Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:11.772112589Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55592023-09-22T23:26:12.007Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.772130185Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55602023-09-22T23:26:12.016Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.772148357Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55612023-09-22T23:26:12.020Z{"msg":"78dec864-2600-4c71-8d63-dbd740e3526c New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.772166551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55622023-09-22T23:26:12.023Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.772184369Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55632023-09-22T23:26:12.026Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:11.772201359Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55642023-09-22T23:26:12.769Z{"msg":"[0] 78dec864-2600-4c71-8d63-dbd740e3526c looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.76830669Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330,"looper":"0"}
55652023-09-22T23:26:12.773Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.768391438Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55662023-09-22T23:26:12.776Z{"msg":"[0] 78dec864-2600-4c71-8d63-dbd740e3526c (8eba43b0-4e73-44df-ad6b-f5657162f6a4) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.769378441Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55672023-09-22T23:26:12.779Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.769432246Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55682023-09-22T23:26:12.783Z{"msg":"[0] client is_active_req TRUE, promote! session 8eba43b0-4e73-44df-ad6b-f5657162f6a4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.769475334Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55692023-09-22T23:26:12.786Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.770072834Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55702023-09-22T23:26:12.789Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.770129166Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55712023-09-22T23:26:12.792Z{"msg":"78dec864-2600-4c71-8d63-dbd740e3526c WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.770171674Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55722023-09-22T23:26:12.796Z{"msg":"[0] 78dec864-2600-4c71-8d63-dbd740e3526c (8eba43b0-4e73-44df-ad6b-f5657162f6a4) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.772587275Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55732023-09-22T23:26:12.799Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.772702115Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55742023-09-22T23:26:12.803Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:12.772762018Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55752023-09-22T23:26:12.806Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.77281037Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55762023-09-22T23:26:12.809Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.77285039Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55772023-09-22T23:26:12.812Z{"msg":"78dec864-2600-4c71-8d63-dbd740e3526c WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.772890452Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55782023-09-22T23:26:12.815Z{"msg":"[0]R flush_numbers[0..12]: [62, 34, 58, 52, 55, 55, 60, 56, 58, 63, 55, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.772930788Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55792023-09-22T23:26:12.819Z{"msg":"[0]R generation[0..12]: [40, 24, 39, 35, 37, 37, 40, 38, 39, 41, 37, 35]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.772985361Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55802023-09-22T23:26:12.822ZThe guest has finished waiting for activation with:42
55812023-09-22T23:26:12.825Z{"msg":"Wait for a query_work_queue command to finish before sending IO
55822023-09-22T23:26:12.828Z[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.773069357Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55832023-09-22T23:26:12.831Z{"msg":"[1]R flush_numbers[0..12]: [62, 34, 58, 52, 55, 55, 60, 56, 58, 63, 55, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.773135686Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55842023-09-22T23:26:12.834Z{"msg":"[1]R generation[0..12]: [40, 24, 39, 35, 37, 37, 40, 38, 39, 41, 37, 35]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.773264283Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55852023-09-22T23:26:12.838Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.773336268Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55862023-09-22T23:26:12.841Z{"msg":"[2]R flush_numbers[0..12]: [62, 34, 58, 52, 55, 55, 60, 56, 58, 63, 55, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.773399892Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55872023-09-22T23:26:12.844Z{"msg":"[2]R generation[0..12]: [40, 24, 39, 35, 37, 37, 40, 38, 39, 41, 37, 35]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.773442277Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55882023-09-22T23:26:12.847Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.773465817Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55892023-09-22T23:26:12.850Z{"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.773523402Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55902023-09-22T23:26:12.853Z{"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.773588948Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55912023-09-22T23:26:12.856Z{"msg":"Next flush: 64","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.773637042Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55922023-09-22T23:26:12.859Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.773683093Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55932023-09-22T23:26:12.862Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.773717874Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55942023-09-22T23:26:12.865Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.773784256Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55952023-09-22T23:26:12.870Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.77384647Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55962023-09-22T23:26:12.873Z{"msg":"78dec864-2600-4c71-8d63-dbd740e3526c is now active with session: 8eba43b0-4e73-44df-ad6b-f5657162f6a4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.773895162Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55972023-09-22T23:26:12.876Z{"msg":"78dec864-2600-4c71-8d63-dbd740e3526c Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.773961009Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55982023-09-22T23:26:12.879Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.77402177Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
55992023-09-22T23:26:12.882Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.774064664Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
56002023-09-22T23:26:12.887Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","levelRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
56012023-09-22T23:26:12.890Z":30,"time":"2023-09-22T23:26:12.774123551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
56022023-09-22T23:26:12.893Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.774189061Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
56032023-09-22T23:26:12.896Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.774253811Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
56042023-09-22T23:26:12.899Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.774315222Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
56052023-09-22T23:26:12.902Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.774377908Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
56062023-09-22T23:26:12.905Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.774439328Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
56072023-09-22T23:26:12.908Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.774498854Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
56082023-09-22T23:26:12.911Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.774557522Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
56092023-09-22T23:26:12.914Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.774617028Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1330}
56102023-09-22T23:26:12.917ZLoading write count information from file "/tmp/test_repair_verify.data"
56112023-09-22T23:26:12.920ZRun Repair workload
56122023-09-22T23:26:12.923Z01/30 Write block 241 len 8192 data: 5 5
56132023-09-22T23:26:12.926Z02/30 Read block 143 len 40960
56142023-09-22T23:26:12.929Z03/30 Read block 422 len 40960
56152023-09-22T23:26:12.932Z04/30 Write block 60 len 28672 data: 6 6 6 7 7 7 7
56162023-09-22T23:26:12.935Z05/30 Flush
56172023-09-22T23:26:12.938Z06/30 Write block 98 len 40960 data: 6 4 3 3 2 3 4 6 6 6
56182023-09-22T23:26:12.941Z07/30 Read block 247 len 4096
56192023-09-22T23:26:12.944Z08/30 Write block 592 len 20480 data: 4 5 4 5 5
56202023-09-22T23:26:12.947Z09/30 Write block 381 len 20480 data: 5 4 4 5 5
56212023-09-22T23:26:12.950Z10/30 Write block 420 len 20480 data: 3 3 2 2 3
56222023-09-22T23:26:12.953Z11/30 Write block 288 len 36864 data: 6 6 7 7 6 6 5 5 4
56232023-09-22T23:26:12.956Z12/30 Flush
56242023-09-22T23:26:12.959Z13/30 Write block 65 len 20480 data: 8 8 8 9 10
56252023-09-22T23:26:12.962Z14/30 Write block 373 len 12288 data: 3 4 4
56262023-09-22T23:26:12.965Z15/30 Read block 271 len 4096
56272023-09-22T23:26:12.968Z16/30 Write block 462 len 16384 data: 3 4 3 4
56282023-09-22T23:26:12.971Z17/30 Write block 413 len 40960 data: 5 6 5 5 4 3 3 4 4 3
56292023-09-22T23:26:12.974Z18/30 Read block 522 len 36864
56302023-09-22T23:26:12.978Z19/30 Read block 513 len 32768
56312023-09-22T23:26:12.981Z20/30 Read block 195 len 36864
56322023-09-22T23:26:12.985Z21/30 Read block 378 len 28672
56332023-09-22T23:26:12.990Z22/30 Flush
56342023-09-22T23:26:12.995Z23/30 Write block 526 len 4096 data: 4
56352023-09-22T23:26:13.002Z24/30 Write block 516 len 20480 data: 5 5 5 4 3
56362023-09-22T23:26:13.005Z25/30 Write block 18 len 4096 data: 5
56372023-09-22T23:26:13.008Z26/30 Write block 441 len 4096 data: 3
56382023-09-22T23:26:13.012Z27/30 Read block 493 len 32768
56392023-09-22T23:26:13.016Z28/30 Write block 409 len 4096 data: 7
56402023-09-22T23:26:13.019Z29/30 Read block 372 len 4096
56412023-09-22T23:26:13.022Z30/30 Read block 210 len 36864
56422023-09-22T23:26:13.025Z----------------------------------------------------------------
56432023-09-22T23:26:13.028Z Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30
56442023-09-22T23:26:13.031ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
56452023-09-22T23:26:13.035Z 1 Acked 1000 Write 2 Sent Done Done false
56462023-09-22T23:26:13.038Z 2 Acked 1001 Read 10 Sent Done Done false
56472023-09-22T23:26:13.041Z 3 Acked 1002 Read 10 Sent Done Done false
56482023-09-22T23:26:13.045Z 4 Acked 1003 Write 7 Sent Done Done false
56492023-09-22T23:26:13.048Z 5 Acked 1004 Flush 0 Sent Done Done false
56502023-09-22T23:26:13.052Z 6 Acked 1005 Write 10 Sent Done Done false
56512023-09-22T23:26:13.056Z 7 Acked 1006 Read 1 Sent Done Done false
56522023-09-22T23:26:13.058Z 8 Acked 1007 Write 5 Sent Done Done false
56532023-09-22T23:26:13.062Z 9 Acked 1008 Write 5 Sent Done Done false
56542023-09-22T23:26:13.065Z 10 Acked 1009 Write 5 Sent Done Done false
56552023-09-22T23:26:13.068Z 11 Acked 1010 Write 9 Sent Done Done false
56562023-09-22T23:26:13.071Z 12 Acked 1011 Flush 0 Sent Done Done false
56572023-09-22T23:26:13.074Z 13 Acked 1012 Write 5 Sent Done Done false
56582023-09-22T23:26:13.077Z 14 Acked 1013 Write 3 Sent Done Done false
56592023-09-22T23:26:13.081Z 15 Acked 1014 Read 1 Sent Done Done false
56602023-09-22T23:26:13.084Z 16 Acked 1015 Write 4 Sent Done Done false
56612023-09-22T23:26:13.087Z 17 Acked 1016 Write 10 Sent Done Done false
56622023-09-22T23:26:13.090Z 18 Acked 1017 Read 9 Sent Done Done false
56632023-09-22T23:26:13.093Z 19 Acked 1018 Read 8 Sent Done Done false
56642023-09-22T23:26:13.096Z 20 Acked 1019 Read 9 Sent Done Done false
56652023-09-22T23:26:13.099Z 21 Acked 1020 Read 7 Sent Done Done false
56662023-09-22T23:26:13.102Z 22 Acked 1021 Flush 0 Sent Done Done false
56672023-09-22T23:26:13.105Z 23 Acked 1022 Write 1 Sent Done Done false
56682023-09-22T23:26:13.109Z 24 Acked 1023 Write 5 Sent Done Done false
56692023-09-22T23:26:13.112Z 25 Acked 1024 Write 1 Sent Done Done false
56702023-09-22T23:26:13.115Z 26 Acked 1025 Write 1 Sent Done Done false
56712023-09-22T23:26:13.118Z 27 Acked 1026 Read 8 Sent Done Done false
56722023-09-22T23:26:13.121Z 28 Acked 1027 Write 1 Sent Done Done false
56732023-09-22T23:26:13.124Z 29 Acked 1028 Read 1 Sent Done Done false
56742023-09-22T23:26:13.127Z 30 Acked 1029 Read 9 Sent Sent Done false
56752023-09-22T23:26:13.130Z STATES DS:0 DS:1 DS:2 TOTAL
56762023-09-22T23:26:13.133Z New 0 0 0 0
56772023-09-22T23:26:13.137Z Sent 30 1 0 31
56782023-09-22T23:26:13.140Z Done 0 29 30 59
56792023-09-22T23:26:13.143Z Skipped 0 0 0 0
56802023-09-22T23:26:13.146Z Error 0 0 0 0
56812023-09-22T23:26:13.149ZLast Flush: 0 1021 1021
56822023-09-22T23:26:13.152ZDownstairs last five completed:
56832023-09-22T23:26:13.197ZUpstairs last five completed: 30 29 28 27 26 25
56842023-09-22T23:26:13.200ZWrote out file "/tmp/test_repair_verify.data"
56852023-09-22T23:26:13.204Z
56862023-09-22T23:26:13.207ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
56872023-09-22T23:26:13.211Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.922281523Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1331}
56882023-09-22T23:26:13.216Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.922534571Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1331}
56892023-09-22T23:26:13.220Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.922557335Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1331}
56902023-09-22T23:26:13.223Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.945914533Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1331}
56912023-09-22T23:26:13.227Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.945948163Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1331}
56922023-09-22T23:26:13.230Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.945967604Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1331}
56932023-09-22T23:26:13.234Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.966269996Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1331}
56942023-09-22T23:26:13.237Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.966304965Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1331}
56952023-09-22T23:26:13.240Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:12.966325045Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1331}
56962023-09-22T23:26:13.244ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
56972023-09-22T23:26:13.247Z 0 000-019 40 40 40 62 62 62 F T T <---
56982023-09-22T23:26:13.250ZError: Difference in extent metadata found!
56992023-09-22T23:26:13.253Z 1 020-039 24 24 24 34 34 34 F F F
57002023-09-22T23:26:13.256Z 2 040-059 39 39 39 58 58 58 F F F
57012023-09-22T23:26:13.260Z 3 060-079 35 42 42 52 66 66 F F F <---
57022023-09-22T23:26:13.263Z 4 080-099 37 42 42 55 65 65 F F F <---
57032023-09-22T23:26:13.266Z 5 100-119 37 42 42 55 65 65 F F F <---
57042023-09-22T23:26:13.269Z 6 120-139 40 40 40 60 60 60 F F F
57052023-09-22T23:26:13.272Z 7 140-159 38 38 38 56 56 56 F F F
57062023-09-22T23:26:13.275Z 8 160-179 39 39 39 58 58 58 F F F
57072023-09-22T23:26:13.278Z 9 180-199 41 41 41 63 63 63 F F F
57082023-09-22T23:26:13.282Z 10 200-219 37 37 37 55 55 55 F F F
57092023-09-22T23:26:13.285Z 11 220-239 35 35 35 52 52 52 F F F
57102023-09-22T23:26:13.288Z 12 240-259 40 42 42 61 64 64 F F F <---
57112023-09-22T23:26:13.291Z 13 260-279 40 40 40 61 61 61 F F F
57122023-09-22T23:26:13.294Z 14 280-299 40 42 42 61 65 65 F F F <---
57132023-09-22T23:26:13.297Z 15 300-319 37 37 37 55 55 55 F F F
57142023-09-22T23:26:13.300Z 16 320-339 36 36 36 53 53 53 F F F
57152023-09-22T23:26:13.303Z 17 340-359 38 38 38 57 57 57 F F F
57162023-09-22T23:26:13.306Z 18 360-379 40 42 42 61 66 66 F F F <---
57172023-09-22T23:26:13.309Z 19 380-399 34 42 42 50 65 65 F F F <---
57182023-09-22T23:26:13.313Z 20 400-419 40 42 42 62 66 66 F T T <---
57192023-09-22T23:26:13.317Z 21 420-439 40 42 42 62 66 66 F F F <---
57202023-09-22T23:26:13.320Z 22 440-459 41 41 41 63 63 63 F T T <---
57212023-09-22T23:26:13.323Z 23 460-479 36 42 42 53 66 66 F F F <---
57222023-09-22T23:26:13.326Z 24 480-499 40 40 40 61 61 61 F F F
57232023-09-22T23:26:13.330Z 25 500-519 40 40 40 60 60 60 F T T <---
57242023-09-22T23:26:13.333Z 26 520-539 39 39 39 58 58 58 F T T <---
57252023-09-22T23:26:13.336Z 27 540-559 41 41 41 63 63 63 F F F
57262023-09-22T23:26:13.339Z 28 560-579 41 41 41 63 63 63 F F F
57272023-09-22T23:26:13.342Z 29 580-599 36 42 42 54 65 65 F F F <---
57282023-09-22T23:26:13.345ZMax gen: 42, Max flush: 66
57292023-09-22T23:26:13.348ZA Difference in extent metadata is expected here
57302023-09-22T23:26:13.352ZOn loop 21
57312023-09-22T23:26:13.355Z
57322023-09-22T23:26:13.359ZVerifying data now
57332023-09-22T23:26:19.584ZNo change after verify
57342023-09-22T23:26:19.588ZLoop: 21 Downstairs dump after verify (and repair):
57352023-09-22T23:26:19.602Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.600235853Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1336}
57362023-09-22T23:26:19.606Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.600618971Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1336}
57372023-09-22T23:26:19.611Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.600643878Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1336}
57382023-09-22T23:26:19.621Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.619709389Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1336}
57392023-09-22T23:26:19.624Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.619762607Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1336}
57402023-09-22T23:26:19.627Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.619783522Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1336}
57412023-09-22T23:26:19.633Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.630857941Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1336}
57422023-09-22T23:26:19.636Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.630910115Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1336}
57432023-09-22T23:26:19.639Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.630930738Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1336}
57442023-09-22T23:26:19.643ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
57452023-09-22T23:26:19.646Z 0 000-019 43 43 43 67 67 67 F F F
57462023-09-22T23:26:19.649Z 1 020-039 24 24 24 34 34 34 F F F
57472023-09-22T23:26:19.653Z 2 040-059 39 39 39 58 58 58 F F F
57482023-09-22T23:26:19.656Z 3 060-079 42 42 42 66 66 66 F F F
57492023-09-22T23:26:19.659Z 4 080-099 42 42 42 65 65 65 F F F
57502023-09-22T23:26:19.663Z 5 100-119 42 42 42 65 65 65 F F F
57512023-09-22T23:26:19.666Z 6 120-139 40 40 40 60 60 60 F F F
57522023-09-22T23:26:19.669Z 7 140-159 38 38 38 56 56 56 F F F
57532023-09-22T23:26:19.672Z 8 160-179 39 39 39 58 58 58 F F F
57542023-09-22T23:26:19.676Z 9 180-199 41 41 41 63 63 63 F F F
57552023-09-22T23:26:19.678Z 10 200-219 37 37 37 55 55 55 F F F
57562023-09-22T23:26:19.689Z 11 220-239 35 35 35 52 52 52 F F F
57572023-09-22T23:26:19.692Z 12 240-259 42 42 42 64 64 64 F F F
57582023-09-22T23:26:19.696Z 13 260-279 40 40 40 61 61 61 F F F
57592023-09-22T23:26:19.699Z 14 280-299 42 42 42 65 65 65 F F F
57602023-09-22T23:26:19.702Z 15 300-319 37 37 37 55 55 55 F F F
57612023-09-22T23:26:19.706Z 16 320-339 36 36 36 53 53 53 F F F
57622023-09-22T23:26:19.709Z 17 340-359 38 38 38 57 57 57 F F F
57632023-09-22T23:26:19.712Z 18 360-379 42 42 42 66 66 66 F F F
57642023-09-22T23:26:19.716Z 19 380-399 42 42 42 65 65 65 F F F
57652023-09-22T23:26:19.719Z 20 400-419 43 43 43 67 67 67 F F F
57662023-09-22T23:26:19.722Z 21 420-439 42 42 42 66 66 66 F F F
57672023-09-22T23:26:19.725Z 22 440-459 43 43 43 67 67 67 F F F
57682023-09-22T23:26:19.729Z 23 460-479 42 42 42 66 66 66 F F F
57692023-09-22T23:26:19.732Z 24 480-499 40 40 40 61 61 61 F F F
57702023-09-22T23:26:19.735Z 25 500-519 43 43 43 67 67 67 F F F
57712023-09-22T23:26:19.739Z 26 520-539 43 43 43 67 67 67 F F F
57722023-09-22T23:26:19.742Z 27 540-559 41 41 41 63 63 63 F F F
57732023-09-22T23:26:19.745Z 28 560-579 41 41 41 63 63 63 F F F
57742023-09-22T23:26:19.748Z 29 580-599 42 42 42 65 65 65 F F F
57752023-09-22T23:26:19.751ZMax gen: 43, Max flush: 67
57762023-09-22T23:26:19.754Z
57772023-09-22T23:26:19.758ZBegin loop 22
57782023-09-22T23:26:19.761ZDownstairs to restart: 1
57792023-09-22T23:26:19.764Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.681448819Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
57802023-09-22T23:26:19.767Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.681829147Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
57812023-09-22T23:26:19.771Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.681889756Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
57822023-09-22T23:26:19.774Z{"msg":"Crucible stats registered with UUID: 46f780bf-be81-4cf2-9299-f7ab396e6009","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.681915699Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
57832023-09-22T23:26:19.777Z{"msg":"Crucible 46f780bf-be81-4cf2-9299-f7ab396e6009 has session id: 98aa90cc-a91b-4e4b-abd0-cc6644c5e50b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.681973539Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
57842023-09-22T23:26:19.780ZCrucible runtime is spawned
57852023-09-22T23:26:19.784ZThe guest has requested activation with gen:44
57862023-09-22T23:26:19.787Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.68226771Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338,"task":"up_listen"}
57872023-09-22T23:26:19.791Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.682348619Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
57882023-09-22T23:26:19.801Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.682436779Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
57892023-09-22T23:26:19.805Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.682506823Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338,"looper":"1"}
57902023-09-22T23:26:19.808Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.682640769Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338,"looper":"2"}
57912023-09-22T23:26:19.811Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.682800805Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338,"looper":"0"}
57922023-09-22T23:26:19.814Z{"msg":"[2] 46f780bf-be81-4cf2-9299-f7ab396e6009 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.682889138Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338,"looper":"2"}
57932023-09-22T23:26:19.818Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.683048189Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
57942023-09-22T23:26:19.821Z{"msg":"46f780bf-be81-4cf2-9299-f7ab396e6009 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.683105512Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
57952023-09-22T23:26:19.824Z{"msg":"Set desired generation to :44","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.683160407Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
57962023-09-22T23:26:19.827Z{"msg":"[0] 46f780bf-be81-4cf2-9299-f7ab396e6009 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.68323712Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338,"looper":"0"}
57972023-09-22T23:26:19.832Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.683294347Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
57982023-09-22T23:26:19.835Z{"msg":"[2] 46f780bf-be81-4cf2-9299-f7ab396e6009 (2deea310-deef-4cd9-be83-a7bfd545711d) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.683781028Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
57992023-09-22T23:26:19.838Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.683951144Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58002023-09-22T23:26:19.841Z{"msg":"[2] client is_active_req TRUE, promote! session 2deea310-deef-4cd9-be83-a7bfd545711d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.68401885Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58012023-09-22T23:26:19.844Z{"msg":"[0] 46f780bf-be81-4cf2-9299-f7ab396e6009 (2deea310-deef-4cd9-be83-a7bfd545711d) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.684087057Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58022023-09-22T23:26:19.847Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.68412388Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58032023-09-22T23:26:19.850Z{"msg":"[0] client is_active_req TRUE, promote! session 2deea310-deef-4cd9-be83-a7bfd545711d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.684156876Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58042023-09-22T23:26:19.854Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.6845854Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58052023-09-22T23:26:19.858Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.684645135Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58062023-09-22T23:26:19.861Z{"msg":"46f780bf-be81-4cf2-9299-f7ab396e6009 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.684686085Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58072023-09-22T23:26:19.864Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.68477944Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58082023-09-22T23:26:19.867Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.684966315Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58092023-09-22T23:26:19.870Z{"msg":"46f780bf-be81-4cf2-9299-f7ab396e6009 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.685040288Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58102023-09-22T23:26:19.874Z{"msg":"[2] 46f780bf-be81-4cf2-9299-f7ab396e6009 (2deea310-deef-4cd9-be83-a7bfd545711d) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.687197774Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58112023-09-22T23:26:19.878Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.687262516Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58122023-09-22T23:26:19.881Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:19.687320113Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58132023-09-22T23:26:19.884Z{"msg":"[0] 46f780bf-be81-4cf2-9299-f7ab396e6009 (2deea310-deef-4cd9-be83-a7bfd545711d) WaitActive New WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.687378651Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58142023-09-22T23:26:19.888Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.68743237Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58152023-09-22T23:26:19.891Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:19.68748597Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58162023-09-22T23:26:19.894Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.687540314Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58172023-09-22T23:26:19.898Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.687593615Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58182023-09-22T23:26:19.901Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.687647829Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58192023-09-22T23:26:19.904Z{"msg":"46f780bf-be81-4cf2-9299-f7ab396e6009 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.687701679Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58202023-09-22T23:26:19.908Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.687754808Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58212023-09-22T23:26:19.912Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.687806818Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58222023-09-22T23:26:19.915Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.687860311Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58232023-09-22T23:26:19.920Z{"msg":"46f780bf-be81-4cf2-9299-f7ab396e6009 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.687889747Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58242023-09-22T23:26:19.924Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.687908747Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58252023-09-22T23:26:19.927Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:19.687925908Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58262023-09-22T23:26:20.685Z{"msg":"[1] 46f780bf-be81-4cf2-9299-f7ab396e6009 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.68393502Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338,"looper":"1"}
58272023-09-22T23:26:20.688Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.684021869Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58282023-09-22T23:26:20.691Z{"msg":"[1] 46f780bf-be81-4cf2-9299-f7ab396e6009 (2deea310-deef-4cd9-be83-a7bfd545711d) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.685138345Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58292023-09-22T23:26:20.695Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.685196236Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58302023-09-22T23:26:20.698Z{"msg":"[1] client is_active_req TRUE, promote! session 2deea310-deef-4cd9-be83-a7bfd545711d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.685226274Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58312023-09-22T23:26:20.701Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.685879209Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58322023-09-22T23:26:20.704Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.685964475Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58332023-09-22T23:26:20.707Z{"msg":"46f780bf-be81-4cf2-9299-f7ab396e6009 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.686021323Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58342023-09-22T23:26:20.710Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.688168162Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58352023-09-22T23:26:20.713Z{"msg":"[1] 46f780bf-be81-4cf2-9299-f7ab396e6009 (2deea310-deef-4cd9-be83-a7bfd545711d) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.688396336Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58362023-09-22T23:26:20.717Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.688453151Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58372023-09-22T23:26:20.722Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:20.688489072Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58382023-09-22T23:26:20.726Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.688552828Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58392023-09-22T23:26:20.729Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.688598208Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58402023-09-22T23:26:20.732Z{"msg":"46f780bf-be81-4cf2-9299-f7ab396e6009 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.688640025Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58412023-09-22T23:26:20.735Z{"msg":"[0]R flush_numbers[0..12]: [67, 34, 58, 66, 65, 65, 60, 56, 58, 63, 55, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.68868032Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58422023-09-22T23:26:20.738Z{"msg":"[0]R generation[0..12]: [43, 24, 39, 42, 42, 42, 40, 38, 39, 41, 37, 35]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.688720406Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58432023-09-22T23:26:20.743Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.688761624Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58442023-09-22T23:26:20.747Z{"msg":"[1]R flush_numbers[0..12]: [67, 34, 58, 66, 65, 65, 60, 56, 58, 63, 55, 52]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:44
58452023-09-22T23:26:20.750ZWait for a query_work_queue command to finish before sending IO
58462023-09-22T23:26:20.753Z,"time":"2023-09-22T23:26:20.688801479Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58472023-09-22T23:26:20.757Z{"msg":"[1]R generation[0..12]: [43, 24, 39, 42, 42, 42, 40, 38, 39, 41, 37, 35]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.688972088Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58482023-09-22T23:26:20.760Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.689037447Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58492023-09-22T23:26:20.763Z{"msg":"[2]R flush_numbers[0..12]: [67, 34, 58, 66, 65, 65, 60, 56, 58, 63, 55, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.689081323Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58502023-09-22T23:26:20.767Z{"msg":"[2]R generation[0..12]: [43, 24, 39, 42, 42, 42, 40, 38, 39, 41, 37, 35]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.689137875Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58512023-09-22T23:26:20.770Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.689179937Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58522023-09-22T23:26:20.774Z{"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.689206077Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58532023-09-22T23:26:20.777Z{"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.689270422Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58542023-09-22T23:26:20.780Z{"msg":"Next flush: 68","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.689336803Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58552023-09-22T23:26:20.783Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.689384319Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58562023-09-22T23:26:20.786Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.689406406Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58572023-09-22T23:26:20.789Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.689448829Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58582023-09-22T23:26:20.793Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.689511364Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58592023-09-22T23:26:20.796Z{"msg":"46f780bf-be81-4cf2-9299-f7ab396e6009 is now active with session: 2deea310-deef-4cd9-be83-a7bfd545711d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.689578353Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58602023-09-22T23:26:20.800Z{"msg":"46f780bf-be81-4cf2-9299-f7ab396e6009 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.689626027Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58612023-09-22T23:26:20.803Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.689648202Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58622023-09-22T23:26:20.807Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.689700352Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58632023-09-22T23:26:20.810Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.689769519Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58642023-09-22T23:26:20.813Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.689833723Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58652023-09-22T23:26:20.816Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.689875373Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58662023-09-22T23:26:20.819Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.689930105Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58672023-09-22T23:26:20.826Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.689971741Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58682023-09-22T23:26:20.831Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.690017005Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58692023-09-22T23:26:20.836Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.69007223Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
58702023-09-22T23:26:20.839Z:1338}
58712023-09-22T23:26:20.842Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.690114664Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58722023-09-22T23:26:20.845Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.690177271Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1338}
58732023-09-22T23:26:20.848ZLoading write count information from file "/tmp/test_repair_verify.data"
58742023-09-22T23:26:20.851ZRun Repair workload
58752023-09-22T23:26:20.854Z01/30 Write block 138 len 8192 data: 5 5
58762023-09-22T23:26:20.857Z02/30 Write block 132 len 40960 data: 9 7 7 6 7 6 6 6 6 6
58772023-09-22T23:26:20.861Z03/30 Read block 53 len 12288
58782023-09-22T23:26:20.864Z04/30 Read block 433 len 12288
58792023-09-22T23:26:20.867Z05/30 Read block 577 len 8192
58802023-09-22T23:26:20.870Z06/30 Write block 504 len 28672 data: 4 4 5 5 4 6 4
58812023-09-22T23:26:20.873Z07/30 Read block 108 len 16384
58822023-09-22T23:26:20.876Z08/30 Write block 117 len 32768 data: 2 2 2 4 4 5 5 5
58832023-09-22T23:26:20.889Z09/30 Read block 10 len 24576
58842023-09-22T23:26:20.892Z10/30 Read block 520 len 8192
58852023-09-22T23:26:20.895Z11/30 Write block 170 len 40960 data: 6 6 6 5 5 5 5 4 4 4
58862023-09-22T23:26:20.898Z12/30 Read block 293 len 4096
58872023-09-22T23:26:20.901Z13/30 Write block 295 len 40960 data: 6 5 3 4 5 5 5 5 5 5
58882023-09-22T23:26:20.905Z14/30 Read block 71 len 8192
58892023-09-22T23:26:20.909Z15/30 Read block 266 len 32768
58902023-09-22T23:26:20.912Z16/30 Flush
58912023-09-22T23:26:20.915Z17/30 Write block 267 len 36864 data: 9 7 8 8 8 8 7 6 6
58922023-09-22T23:26:20.918Z18/30 Write block 323 len 8192 data: 6 6
58932023-09-22T23:26:20.922Z19/30 Write block 368 len 8192 data: 3 2
58942023-09-22T23:26:20.925Z20/30 Read block 442 len 40960
58952023-09-22T23:26:20.929Z21/30 Read block 222 len 32768
58962023-09-22T23:26:20.932Z22/30 Read block 310 len 36864
58972023-09-22T23:26:20.935Z23/30 Write block 564 len 32768 data: 4 3 3 6 6 6 3 3
58982023-09-22T23:26:20.939Z24/30 Write block 543 len 8192 data: 5 5
58992023-09-22T23:26:20.942Z25/30 Write block 476 len 12288 data: 4 5 5
59002023-09-22T23:26:20.945Z26/30 Read block 230 len 8192
59012023-09-22T23:26:20.949Z27/30 Read block 341 len 4096
59022023-09-22T23:26:20.952Z28/30 Write block 278 len 16384 data: 8 8 7 6
59032023-09-22T23:26:20.978Z29/30 Read block 291 len 40960
59042023-09-22T23:26:20.981Z30/30 Read block 197 len 4096
59052023-09-22T23:26:20.984Z----------------------------------------------------------------
59062023-09-22T23:26:20.988Z Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30
59072023-09-22T23:26:20.991ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
59082023-09-22T23:26:20.994Z 1 Acked 1000 Write 2 Done Done Done false
59092023-09-22T23:26:20.997Z 2 Acked 1001 Write 10 Done Done Done false
59102023-09-22T23:26:21.000Z 3 Acked 1002 Read 3 Done Done Done false
59112023-09-22T23:26:21.003Z 4 Acked 1003 Read 3 Done Done Done false
59122023-09-22T23:26:21.006Z 5 Acked 1004 Read 2 Done Done Done false
59132023-09-22T23:26:21.010Z 6 Acked 1005 Write 7 Done Done Done false
59142023-09-22T23:26:21.013Z 7 Acked 1006 Read 4 Done Sent Done false
59152023-09-22T23:26:21.016Z 8 Acked 1007 Write 8 Done Sent Done false
59162023-09-22T23:26:21.019Z 9 Acked 1008 Read 6 Done Sent Done false
59172023-09-22T23:26:21.022Z 10 Acked 1009 Read 2 Done Sent Done false
59182023-09-22T23:26:21.025Z 11 Acked 1010 Write 10 Done Sent Done false
59192023-09-22T23:26:21.028Z 12 Acked 1011 Read 1 Done Sent Done false
59202023-09-22T23:26:21.032Z 13 Acked 1012 Write 10 Done Sent Done false
59212023-09-22T23:26:21.035Z 14 Acked 1013 Read 2 Done Sent Done false
59222023-09-22T23:26:21.038Z 15 Acked 1014 Read 8 Done Sent Done false
59232023-09-22T23:26:21.041Z 16 Acked 1015 Flush 0 Done Sent Done false
59242023-09-22T23:26:21.045Z 17 Acked 1016 Write 9 Done Sent Done false
59252023-09-22T23:26:21.049Z 18 Acked 1017 Write 2 Done Sent Done false
59262023-09-22T23:26:21.052Z 19 Acked 1018 Write 2 Done Sent Done false
59272023-09-22T23:26:21.055Z 20 Acked 1019 Read 10 Done Sent Done false
59282023-09-22T23:26:21.059Z 21 Acked 1020 Read 8 Done Sent Done false
59292023-09-22T23:26:21.064Z 22 Acked 1021 Read 9 Done Sent Done false
59302023-09-22T23:26:21.068Z 23 Acked 1022 Write 8 Done Sent Done false
59312023-09-22T23:26:21.073Z 24 Acked 1023 Write 2 Done Sent Done false
59322023-09-22T23:26:21.076Z 25 Acked 1024 Write 3 Done Sent Done false
59332023-09-22T23:26:21.079Z 26 Acked 1025 Read 2 Done Sent Done false
59342023-09-22T23:26:21.082Z 27 Acked 1026 Read 1 Done Sent Done false
59352023-09-22T23:26:21.085Z 28 Acked 1027 Write 4 Done Sent Done false
59362023-09-22T23:26:21.088Z 29 Acked 1028 Read 10 Done Sent Done false
59372023-09-22T23:26:21.091Z 30 Acked 1029 Read 1 Sent Sent Done false
59382023-09-22T23:26:21.094Z STATES DS:0 DS:1 DS:2 TOTAL
59392023-09-22T23:26:21.098Z New 0 0 0 0
59402023-09-22T23:26:21.100Z Sent 1 24 0 25
59412023-09-22T23:26:21.104Z Done 29 6 30 65
59422023-09-22T23:26:21.106Z Skipped 0 0 0 0
59432023-09-22T23:26:21.109Z Error 0 0 0 0
59442023-09-22T23:26:21.113ZLast Flush: 1015 0 1015
59452023-09-22T23:26:21.116ZDownstairs last five completed:
59462023-09-22T23:26:21.119ZUpstairs last five completed: 30 29 28 27 26 25
59472023-09-22T23:26:21.122ZWrote out file "/tmp/test_repair_verify.data"
59482023-09-22T23:26:21.125Z
59492023-09-22T23:26:21.128ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
59502023-09-22T23:26:21.131Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.823837748Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1339}
59512023-09-22T23:26:21.135Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.824214966Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1339}
59522023-09-22T23:26:21.138Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.82424088Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1339}
59532023-09-22T23:26:21.141Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.853657134Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1339}
59542023-09-22T23:26:21.144ZError: Difference in extent metadata found!
59552023-09-22T23:26:21.147Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.85370776Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1339}
59562023-09-22T23:26:21.150Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.853732566Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1339}
59572023-09-22T23:26:21.153Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.881847509Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1339}
59582023-09-22T23:26:21.157Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.881879467Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1339}
59592023-09-22T23:26:21.160Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:20.881898377Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1339}
59602023-09-22T23:26:21.163ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
59612023-09-22T23:26:21.167Z 0 000-019 43 43 43 67 67 67 F F F
59622023-09-22T23:26:21.170Z 1 020-039 24 24 24 34 34 34 F F F
59632023-09-22T23:26:21.173Z 2 040-059 39 39 39 58 58 58 F F F
59642023-09-22T23:26:21.176Z 3 060-079 42 42 42 66 66 66 F F F
59652023-09-22T23:26:21.179Z 4 080-099 42 42 42 65 65 65 F F F
59662023-09-22T23:26:21.182Z 5 100-119 44 42 44 68 65 68 F F F <---
59672023-09-22T23:26:21.185Z 6 120-139 44 40 44 68 60 68 F T F <---
59682023-09-22T23:26:21.188Z 7 140-159 44 38 44 68 56 68 F T F <---
59692023-09-22T23:26:21.193Z 8 160-179 44 39 44 68 58 68 F F F <---
59702023-09-22T23:26:21.198Z 9 180-199 41 41 41 63 63 63 F F F
59712023-09-22T23:26:21.201Z 10 200-219 37 37 37 55 55 55 F F F
59722023-09-22T23:26:21.204Z 11 220-239 35 35 35 52 52 52 F F F
59732023-09-22T23:26:21.207Z 12 240-259 42 42 42 64 64 64 F F F
59742023-09-22T23:26:21.210Z 13 260-279 40 40 40 61 61 61 T F T <---
59752023-09-22T23:26:21.213Z 14 280-299 44 42 44 68 65 68 T F T <---
59762023-09-22T23:26:21.216Z 15 300-319 44 37 44 68 55 68 F F F <---
59772023-09-22T23:26:21.219Z 16 320-339 36 36 36 53 53 53 T F T <---
59782023-09-22T23:26:21.222Z 17 340-359 38 38 38 57 57 57 F F F
59792023-09-22T23:26:21.225Z 18 360-379 42 42 42 66 66 66 T F T <---
59802023-09-22T23:26:21.228Z 19 380-399 42 42 42 65 65 65 F F F
59812023-09-22T23:26:21.232Z 20 400-419 43 43 43 67 67 67 F F F
59822023-09-22T23:26:21.236Z 21 420-439 42 42 42 66 66 66 F F F
59832023-09-22T23:26:21.239Z 22 440-459 43 43 43 67 67 67 F F F
59842023-09-22T23:26:21.242Z 23 460-479 42 42 42 66 66 66 T F T <---
59852023-09-22T23:26:21.248Z 24 480-499 40 40 40 61 61 61 F F F
59862023-09-22T23:26:21.251Z 25 500-519 44 43 44 68 67 68 F T F <---
59872023-09-22T23:26:21.254Z 26 520-539 43 43 43 67 67 67 F F F
59882023-09-22T23:26:21.257Z 27 540-559 41 41 41 63 63 63 T F T <---
59892023-09-22T23:26:21.260Z 28 560-579 41 41 41 63 63 63 T F T <---
59902023-09-22T23:26:21.263Z 29 580-599 42 42 42 65 65 65 F F F
59912023-09-22T23:26:21.267ZMax gen: 44, Max flush: 68
59922023-09-22T23:26:21.270ZA Difference in extent metadata is expected here
59932023-09-22T23:26:21.273ZOn loop 22
59942023-09-22T23:26:21.277Z
59952023-09-22T23:26:21.282ZVerifying data now
59962023-09-22T23:26:27.323ZNo change after verify
59972023-09-22T23:26:27.326ZLoop: 22 Downstairs dump after verify (and repair):
59982023-09-22T23:26:27.342Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.340277187Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1344}
59992023-09-22T23:26:27.346Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.340656152Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1344}
60002023-09-22T23:26:27.349Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.340681128Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1344}
60012023-09-22T23:26:27.360Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.359510245Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1344}
60022023-09-22T23:26:27.363Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.359563404Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1344}
60032023-09-22T23:26:27.366Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.359583996Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1344}
60042023-09-22T23:26:27.371Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.370780532Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1344}
60052023-09-22T23:26:27.375Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.37083103Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1344}
60062023-09-22T23:26:27.378Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.370851455Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1344}
60072023-09-22T23:26:27.384ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
60082023-09-22T23:26:27.386Z 0 000-019 43 43 43 67 67 67 F F F
60092023-09-22T23:26:27.390Z 1 020-039 24 24 24 34 34 34 F F F
60102023-09-22T23:26:27.393Z 2 040-059 39 39 39 58 58 58 F F F
60112023-09-22T23:26:27.396Z 3 060-079 42 42 42 66 66 66 F F F
60122023-09-22T23:26:27.399Z 4 080-099 42 42 42 65 65 65 F F F
60132023-09-22T23:26:27.402Z 5 100-119 44 44 44 68 68 68 F F F
60142023-09-22T23:26:27.404Z 6 120-139 44 44 44 68 68 68 F F F
60152023-09-22T23:26:27.407Z 7 140-159 44 44 44 68 68 68 F F F
60162023-09-22T23:26:27.411Z 8 160-179 44 44 44 68 68 68 F F F
60172023-09-22T23:26:27.415Z 9 180-199 41 41 41 63 63 63 F F F
60182023-09-22T23:26:27.419Z 10 200-219 37 37 37 55 55 55 F F F
60192023-09-22T23:26:27.422Z 11 220-239 35 35 35 52 52 52 F F F
60202023-09-22T23:26:27.425Z 12 240-259 42 42 42 64 64 64 F F F
60212023-09-22T23:26:27.428Z 13 260-279 45 45 45 69 69 69 F F F
60222023-09-22T23:26:27.430Z 14 280-299 45 45 45 69 69 69 F F F
60232023-09-22T23:26:27.433Z 15 300-319 44 44 44 68 68 68 F F F
60242023-09-22T23:26:27.436Z 16 320-339 45 45 45 69 69 69 F F F
60252023-09-22T23:26:27.439Z 17 340-359 38 38 38 57 57 57 F F F
60262023-09-22T23:26:27.442Z 18 360-379 45 45 45 69 69 69 F F F
60272023-09-22T23:26:27.445Z 19 380-399 42 42 42 65 65 65 F F F
60282023-09-22T23:26:27.448Z 20 400-419 43 43 43 67 67 67 F F F
60292023-09-22T23:26:27.451Z 21 420-439 42 42 42 66 66 66 F F F
60302023-09-22T23:26:27.454Z 22 440-459 43 43 43 67 67 67 F F F
60312023-09-22T23:26:27.457Z 23 460-479 45 45 45 69 69 69 F F F
60322023-09-22T23:26:27.460Z 24 480-499 40 40 40 61 61 61 F F F
60332023-09-22T23:26:27.463Z 25 500-519 44 44 44 68 68 68 F F F
60342023-09-22T23:26:27.466Z 26 520-539 43 43 43 67 67 67 F F F
60352023-09-22T23:26:27.471Z 27 540-559 45 45 45 69 69 69 F F F
60362023-09-22T23:26:27.475Z 28 560-579 45 45 45 69 69 69 F F F
60372023-09-22T23:26:27.478Z 29 580-599 42 42 42 65 65 65 F F F
60382023-09-22T23:26:27.481ZMax gen: 45, Max flush: 69
60392023-09-22T23:26:27.484Z
60402023-09-22T23:26:27.487ZBegin loop 23
60412023-09-22T23:26:27.490ZDownstairs to restart: 0
60422023-09-22T23:26:27.493Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.41830235Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60432023-09-22T23:26:27.496Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.418675509Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60442023-09-22T23:26:27.499Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.418728281Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60452023-09-22T23:26:27.502Z{"msg":"Crucible stats registered with UUID: e3b53f0c-4395-4ecb-aac9-28fd0b295628","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.418771711Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60462023-09-22T23:26:27.505Z{"msg":"Crucible e3b53f0c-4395-4ecb-aac9-28fd0b295628 has session id: 795a992f-cce0-4744-be36-b8a72fa379de","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.418818295Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60472023-09-22T23:26:27.508ZCrucible runtime is spawned
60482023-09-22T23:26:27.511ZThe guest has requested activation with gen:46
60492023-09-22T23:26:27.514Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.419136323Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346,"task":"up_listen"}
60502023-09-22T23:26:27.517Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.419214103Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60512023-09-22T23:26:27.520Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.41924004Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346,"looper":"0"}
60522023-09-22T23:26:27.523Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.419267351Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60532023-09-22T23:26:27.526Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.419408032Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346,"looper":"1"}
60542023-09-22T23:26:27.529Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.419617175Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346,"looper":"2"}
60552023-09-22T23:26:27.532Z{"msg":"[1] e3b53f0c-4395-4ecb-aac9-28fd0b295628 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.419712949Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346,"looper":"1"}
60562023-09-22T23:26:27.536Z{"msg":"[2] e3b53f0c-4395-4ecb-aac9-28fd0b295628 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.419800278Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346,"looper":"2"}
60572023-09-22T23:26:27.539Z{"msg":"e3b53f0c-4395-4ecb-aac9-28fd0b295628 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.420054815Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60582023-09-22T23:26:27.542Z{"msg":"Set desired generation to :46","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.420110541Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60592023-09-22T23:26:27.545Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.42021816Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60602023-09-22T23:26:27.549Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.420517427Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60612023-09-22T23:26:27.552Z{"msg":"[1] e3b53f0c-4395-4ecb-aac9-28fd0b295628 (7f9bacbd-91e1-4387-8eae-11ff23c66ed3) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.420858468Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60622023-09-22T23:26:27.555Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.420918704Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60632023-09-22T23:26:27.558Z{"msg":"[1] client is_active_req TRUE, promote! session 7f9bacbd-91e1-4387-8eae-11ff23c66ed3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.421030362Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60642023-09-22T23:26:27.561Z{"msg":"[2] e3b53f0c-4395-4ecb-aac9-28fd0b295628 (7f9bacbd-91e1-4387-8eae-11ff23c66ed3) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.421107807Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60652023-09-22T23:26:27.564Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.421224785Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60662023-09-22T23:26:27.567Z{"msg":"[2] client is_active_req TRUE, promote! session 7f9bacbd-91e1-4387-8eae-11ff23c66ed3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.421319095Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60672023-09-22T23:26:27.573Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.421635177Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60682023-09-22T23:26:27.575Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.421754595Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60692023-09-22T23:26:27.578Z{"msg":"e3b53f0c-4395-4ecb-aac9-28fd0b295628 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.421837233Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60702023-09-22T23:26:27.581Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.42191274Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60712023-09-22T23:26:27.584Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.421993495Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60722023-09-22T23:26:27.587Z{"msg":"e3b53f0c-4395-4ecb-aac9-28fd0b295628 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.422069105Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60732023-09-22T23:26:27.590Z{"msg":"[1] e3b53f0c-4395-4ecb-aac9-28fd0b295628 (7f9bacbd-91e1-4387-8eae-11ff23c66ed3) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.424441617Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60742023-09-22T23:26:27.593Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.424670119Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60752023-09-22T23:26:27.597Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:27.424745799Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60762023-09-22T23:26:27.600Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.424821289Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60772023-09-22T23:26:27.603Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.424893832Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60782023-09-22T23:26:27.605Z{"msg":"e3b53f0c-4395-4ecb-aac9-28fd0b295628 New WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.424979246Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60792023-09-22T23:26:27.608Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.425055045Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60802023-09-22T23:26:27.611Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.425105561Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60812023-09-22T23:26:27.614Z{"msg":"[2] e3b53f0c-4395-4ecb-aac9-28fd0b295628 (7f9bacbd-91e1-4387-8eae-11ff23c66ed3) New WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.425149125Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60822023-09-22T23:26:27.617Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.425188869Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60832023-09-22T23:26:27.620Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:27.425227415Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60842023-09-22T23:26:27.623Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.425265535Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60852023-09-22T23:26:27.626Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.42530401Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60862023-09-22T23:26:27.629Z{"msg":"e3b53f0c-4395-4ecb-aac9-28fd0b295628 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.425342895Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60872023-09-22T23:26:27.632Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.425362795Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60882023-09-22T23:26:27.635Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:27.425379805Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60892023-09-22T23:26:28.423Z{"msg":"[0] e3b53f0c-4395-4ecb-aac9-28fd0b295628 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.421079808Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346,"looper":"0"}
60902023-09-22T23:26:28.427Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.421164124Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60912023-09-22T23:26:28.429Z{"msg":"[0] e3b53f0c-4395-4ecb-aac9-28fd0b295628 (7f9bacbd-91e1-4387-8eae-11ff23c66ed3) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.422364665Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60922023-09-22T23:26:28.432Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.422436466Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60932023-09-22T23:26:28.436Z{"msg":"[0] client is_active_req TRUE, promote! session 7f9bacbd-91e1-4387-8eae-11ff23c66ed3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.422480747Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60942023-09-22T23:26:28.445Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.423141462Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60952023-09-22T23:26:28.448Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.423201838Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60962023-09-22T23:26:28.451Z{"msg":"e3b53f0c-4395-4ecb-aac9-28fd0b295628 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.423245109Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60972023-09-22T23:26:28.455Z{"msg":"[0] e3b53f0c-4395-4ecb-aac9-28fd0b295628 (7f9bacbd-91e1-4387-8eae-11ff23c66ed3) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.425788821Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60982023-09-22T23:26:28.458Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.425861933Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
60992023-09-22T23:26:28.461Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:28.425892196Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61002023-09-22T23:26:28.468Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.425933152Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61012023-09-22T23:26:28.479Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.426029849Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61022023-09-22T23:26:28.482Z{"msg":"e3b53f0c-4395-4ecb-aac9-28fd0b295628 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.426089843Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61032023-09-22T23:26:28.485Z{"msg":"[0]R flush_numbers[0..12]: [67, 34, 58, 66, 65, 68, 68, 68, 68, 63, 55, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.426133296Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61042023-09-22T23:26:28.489Z{"msg":"[0]R generation[0..12]: [43, 24, 39, 42, 42, 44, 44, 44, 44, 41, 37, 35]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.426154833Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61052023-09-22T23:26:28.491Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.426174663Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61062023-09-22T23:26:28.495Z{"msg":"[1]R flush_numbers[0..12]: [67, 34, 58, 66, 65, 68, 68, 68, 68, 63, 55, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.426228808Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61072023-09-22T23:26:28.497Z{"msg":"[1]R generation[0..12]: [43, 24, 39, 42, 42, 44, 44, 44, 44, 41, 37, 35]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.426272434Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61082023-09-22T23:26:28.501Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.426313184Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61092023-09-22T23:26:28.504Z{"msg":"[2]R flush_numbers[0..12]: [67, 34, 58, 66, 65, 68, 68, 68, 68, 63, 55, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.426335414Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61102023-09-22T23:26:28.512Z{"msg":"[2]R generation[0..12]: [43, 24, 39, 42, 42, 44, 44, 44, 44, 41, 37, 35]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.426373066Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61112023-09-22T23:26:28.515Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level"The guest has finished waiting for activation with:46
61122023-09-22T23:26:28.518Z:30Wait for a query_work_queue command to finish before sending IO
61132023-09-22T23:26:28.521Z,"time":"2023-09-22T23:26:28.426400551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61142023-09-22T23:26:28.525Z{"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.426420324Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61152023-09-22T23:26:28.533Z{"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.426480882Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61162023-09-22T23:26:28.536Z{"msg":"Next flush: 70","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.426579073Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61172023-09-22T23:26:28.539Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.426654815Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61182023-09-22T23:26:28.542Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.426734188Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61192023-09-22T23:26:28.546Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.426798284Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61202023-09-22T23:26:28.549Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.426842497Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61212023-09-22T23:26:28.552Z{"msg":"e3b53f0c-4395-4ecb-aac9-28fd0b295628 is now active with session: 7f9bacbd-91e1-4387-8eae-11ff23c66ed3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.426898789Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61222023-09-22T23:26:28.555Z{"msg":"e3b53f0c-4395-4ecb-aac9-28fd0b295628 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.426963058Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61232023-09-22T23:26:28.559Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.427027834Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61242023-09-22T23:26:28.562Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.427092846Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61252023-09-22T23:26:28.565Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.427158067Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61262023-09-22T23:26:28.568Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.42724367Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61272023-09-22T23:26:28.571Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
61282023-09-22T23:26:28.574Z,"time":"2023-09-22T23:26:28.427312531Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61292023-09-22T23:26:28.577Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.427401736Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61302023-09-22T23:26:28.580Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.427448806Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61312023-09-22T23:26:28.584Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.427487486Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61322023-09-22T23:26:28.587Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.427525142Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61332023-09-22T23:26:28.590Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.427563093Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61342023-09-22T23:26:28.593Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.427600089Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1346}
61352023-09-22T23:26:28.596ZLoading write count information from file "/tmp/test_repair_verify.data"
61362023-09-22T23:26:28.599ZRun Repair workload
61372023-09-22T23:26:28.602Z01/30 Write block 396 len 40960 data: 4 4 5 5 6 5 7 6 6 6
61382023-09-22T23:26:28.605Z02/30 Read block 377 len 24576
61392023-09-22T23:26:28.608Z03/30 Write block 196 len 20480 data: 6 6 3 3 3
61402023-09-22T23:26:28.611Z04/30 Write block 473 len 16384 data: 2 3 4 5
61412023-09-22T23:26:28.614Z05/30 Write block 593 len 20480 data: 6 5 6 6 3
61422023-09-22T23:26:28.617Z06/30 Read block 463 len 28672
61432023-09-22T23:26:28.620Z07/30 Write block 429 len 28672 data: 6 6 6 8 7 6 5
61442023-09-22T23:26:28.624Z08/30 Write block 479 len 8192 data: 5 4
61452023-09-22T23:26:28.627Z09/30 Read block 173 len 40960
61462023-09-22T23:26:28.630Z10/30 Read block 1 len 20480
61472023-09-22T23:26:28.633Z11/30 Write block 34 len 24576 data: 3 4 7 7 7 7
61482023-09-22T23:26:28.635Z12/30 Write block 388 len 32768 data: 6 6 6 5 4 3 3 4
61492023-09-22T23:26:28.638Z13/30 Write block 305 len 24576 data: 4 4 4 4 3 2
61502023-09-22T23:26:28.641Z14/30 Read block 441 len 8192
61512023-09-22T23:26:28.644Z15/30 Write block 304 len 20480 data: 6 5 5 5 5
61522023-09-22T23:26:28.648Z16/30 Read block 242 len 40960
61532023-09-22T23:26:28.651Z17/30 Read block 57 len 40960
61542023-09-22T23:26:28.654Z18/30 Read block 552 len 40960
61552023-09-22T23:26:28.657Z19/30 Read block 500 len 32768
61562023-09-22T23:26:28.660Z20/30 Read block 112 len 36864
61572023-09-22T23:26:28.663Z21/30 Read block 446 len 28672
61582023-09-22T23:26:28.666Z22/30 Read block 356 len 4096
61592023-09-22T23:26:28.670Z23/30 Flush
61602023-09-22T23:26:28.673Z24/30 Flush
61612023-09-22T23:26:28.677Z25/30 Write block 159 len 4096 data: 6
61622023-09-22T23:26:28.680Z26/30 Write block 461 len 40960 data: 5 4 5 4 5 3 3 3 4 2
61632023-09-22T23:26:28.683Z27/30 Read block 37 len 36864
61642023-09-22T23:26:28.686Z28/30 Read block 516 len 36864
61652023-09-22T23:26:28.688Z29/30 Write block 335 len 40960 data: 4 4 4 4 4 2 2 4 4 5
61662023-09-22T23:26:28.693Z30/30 Write block 519 len 36864 data: 5 4 2 3 3 2 4 5 5
61672023-09-22T23:26:28.697Z----------------------------------------------------------------
61682023-09-22T23:26:28.700Z Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30
61692023-09-22T23:26:28.703ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
61702023-09-22T23:26:28.706Z 1 Acked 1000 Write 10 Done Done Done false
61712023-09-22T23:26:28.709Z 2 Acked 1001 Read 6 Done Done Done false
61722023-09-22T23:26:28.712Z 3 Acked 1002 Write 5 Done Done Done false
61732023-09-22T23:26:28.715Z 4 Acked 1003 Write 4 Done Done Done false
61742023-09-22T23:26:28.718Z 5 Acked 1004 Write 5 Done Done Done false
61752023-09-22T23:26:28.721Z 6 Acked 1005 Read 7 Done Done Done false
61762023-09-22T23:26:28.724Z 7 Acked 1006 Write 7 Sent Done Done false
61772023-09-22T23:26:28.727Z 8 Acked 1007 Write 2 Sent Done Done false
61782023-09-22T23:26:28.730Z 9 Acked 1008 Read 10 Sent Done Done false
61792023-09-22T23:26:28.733Z 10 Acked 1009 Read 5 Sent Done Done false
61802023-09-22T23:26:28.736Z 11 Acked 1010 Write 6 Sent Done Done false
61812023-09-22T23:26:28.739Z 12 Acked 1011 Write 8 Sent Done Done false
61822023-09-22T23:26:28.742Z 13 Acked 1012 Write 6 Sent Done Done false
61832023-09-22T23:26:28.745Z 14 Acked 1013 Read 2 Sent Done Done false
61842023-09-22T23:26:28.748Z 15 Acked 1014 Write 5 Sent Done Done false
61852023-09-22T23:26:28.751Z 16 Acked 1015 Read 10 Sent Done Done false
61862023-09-22T23:26:28.754Z 17 Acked 1016 Read 10 Sent Done Done false
61872023-09-22T23:26:28.757Z 18 Acked 1017 Read 10 Sent Done Done false
61882023-09-22T23:26:28.760Z 19 Acked 1018 Read 8 Sent Done Done false
61892023-09-22T23:26:28.763Z 20 Acked 1019 Read 9 Sent Done Done false
61902023-09-22T23:26:28.766Z 21 Acked 1020 Read 7 Sent Done Done false
61912023-09-22T23:26:28.769Z 22 Acked 1021 Read 1 Sent Done Done false
61922023-09-22T23:26:28.772Z 23 Acked 1022 Flush 0 Sent Done Done false
61932023-09-22T23:26:28.775Z 24 Acked 1023 Flush 0 Sent Done Done false
61942023-09-22T23:26:28.778Z 25 Acked 1024 Write 1 Sent Done Done false
61952023-09-22T23:26:28.781Z 26 Acked 1025 Write 10 Sent Done Done false
61962023-09-22T23:26:28.784Z 27 Acked 1026 Read 9 Sent Done Done false
61972023-09-22T23:26:28.786Z 28 Acked 1027 Read 9 Sent Done Done false
61982023-09-22T23:26:28.789Z 29 Acked 1028 Write 10 Sent Sent Sent false
61992023-09-22T23:26:28.793Z 30 Acked 1029 Write 9 Sent Sent Sent false
62002023-09-22T23:26:28.796Z STATES DS:0 DS:1 DS:2 TOTAL
62012023-09-22T23:26:28.799Z New 0 0 0 0
62022023-09-22T23:26:28.803Z Sent 24 2 2 28
62032023-09-22T23:26:28.807Z Done 6 28 28 62
62042023-09-22T23:26:28.810Z Skipped 0 0 0 0
62052023-09-22T23:26:28.813Z Error 0 0 0 0
62062023-09-22T23:26:28.816ZLast Flush: 0 1023 1023
62072023-09-22T23:26:28.819ZDownstairs last five completed:
62082023-09-22T23:26:28.822ZUpstairs last five completed: 30 29 28 27 26 25
62092023-09-22T23:26:28.826ZWrote out file "/tmp/test_repair_verify.data"
62102023-09-22T23:26:28.829Z
62112023-09-22T23:26:28.832ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
62122023-09-22T23:26:28.834Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.569082567Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1347}
62132023-09-22T23:26:28.838Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.569429964Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1347}
62142023-09-22T23:26:28.841Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.569453631Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1347}
62152023-09-22T23:26:28.844Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.619580418Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1347}
62162023-09-22T23:26:28.848Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.619626484Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1347}
62172023-09-22T23:26:28.850Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.619666189Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1347}
62182023-09-22T23:26:28.853Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.639706023Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1347}
62192023-09-22T23:26:28.856Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.639736523Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1347}
62202023-09-22T23:26:28.860Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:28.639759737Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1347}
62212023-09-22T23:26:28.863ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
62222023-09-22T23:26:28.866ZError: Difference in extent metadata found!
62232023-09-22T23:26:28.869Z 0 000-019 43 43 43 67 67 67 F F F
62242023-09-22T23:26:28.871Z 1 020-039 24 46 46 34 70 70 F F F <---
62252023-09-22T23:26:28.874Z 2 040-059 39 39 39 58 58 58 F F F
62262023-09-22T23:26:28.877Z 3 060-079 42 42 42 66 66 66 F F F
62272023-09-22T23:26:28.880Z 4 080-099 42 42 42 65 65 65 F F F
62282023-09-22T23:26:28.883Z 5 100-119 44 44 44 68 68 68 F F F
62292023-09-22T23:26:28.886Z 6 120-139 44 44 44 68 68 68 F F F
62302023-09-22T23:26:28.889Z 7 140-159 44 44 44 68 68 68 F T T <---
62312023-09-22T23:26:28.892Z 8 160-179 44 44 44 68 68 68 F F F
62322023-09-22T23:26:28.895Z 9 180-199 41 46 46 63 70 70 T F F <---
62332023-09-22T23:26:28.898Z 10 200-219 37 46 46 55 70 70 T F F <---
62342023-09-22T23:26:28.902Z 11 220-239 35 35 35 52 52 52 F F F
62352023-09-22T23:26:28.905Z 12 240-259 42 42 42 64 64 64 F F F
62362023-09-22T23:26:28.908Z 13 260-279 45 45 45 69 69 69 F F F
62372023-09-22T23:26:28.911Z 14 280-299 45 45 45 69 69 69 F F F
62382023-09-22T23:26:28.914Z 15 300-319 44 46 46 68 70 70 F F F <---
62392023-09-22T23:26:28.917Z 16 320-339 45 45 45 69 69 69 F T T <---
62402023-09-22T23:26:28.921Z 17 340-359 38 38 38 57 57 57 F T T <---
62412023-09-22T23:26:28.924Z 18 360-379 45 45 45 69 69 69 F F F
62422023-09-22T23:26:28.927Z 19 380-399 42 46 46 65 70 70 T F F <---
62432023-09-22T23:26:28.930Z 20 400-419 43 46 46 67 70 70 T F F <---
62442023-09-22T23:26:28.933Z 21 420-439 42 46 46 66 70 70 F F F <---
62452023-09-22T23:26:28.935Z 22 440-459 43 43 43 67 67 67 F F F
62462023-09-22T23:26:28.938Z 23 460-479 45 46 46 69 70 70 T T T <---
62472023-09-22T23:26:28.941Z 24 480-499 40 46 46 61 70 70 F F F <---
62482023-09-22T23:26:28.944Z 25 500-519 44 44 44 68 68 68 F T T <---
62492023-09-22T23:26:28.948Z 26 520-539 43 43 43 67 67 67 F T T <---
62502023-09-22T23:26:28.951Z 27 540-559 45 45 45 69 69 69 F F F
62512023-09-22T23:26:28.954Z 28 560-579 45 45 45 69 69 69 F F F
62522023-09-22T23:26:28.957Z 29 580-599 42 46 46 65 70 70 T F F <---
62532023-09-22T23:26:28.960ZMax gen: 46, Max flush: 70
62542023-09-22T23:26:28.963ZA Difference in extent metadata is expected here
62552023-09-22T23:26:28.966ZOn loop 23
62562023-09-22T23:26:28.970Z
62572023-09-22T23:26:28.973ZVerifying data now
62582023-09-22T23:26:35.243ZNo change after verify
62592023-09-22T23:26:35.246ZLoop: 23 Downstairs dump after verify (and repair):
62602023-09-22T23:26:35.281Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.260050903Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1352}
62612023-09-22T23:26:35.284Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.260443185Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1352}
62622023-09-22T23:26:35.287Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.26046748Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1352}
62632023-09-22T23:26:35.290Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.27953951Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1352}
62642023-09-22T23:26:35.292Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.279597254Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1352}
62652023-09-22T23:26:35.295Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.279637005Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1352}
62662023-09-22T23:26:35.298Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.290736149Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1352}
62672023-09-22T23:26:35.301Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.290786974Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1352}
62682023-09-22T23:26:35.304Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.290824765Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1352}
62692023-09-22T23:26:35.307ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
62702023-09-22T23:26:35.310Z 0 000-019 43 43 43 67 67 67 F F F
62712023-09-22T23:26:35.312Z 1 020-039 46 46 46 70 70 70 F F F
62722023-09-22T23:26:35.315Z 2 040-059 39 39 39 58 58 58 F F F
62732023-09-22T23:26:35.318Z 3 060-079 42 42 42 66 66 66 F F F
62742023-09-22T23:26:35.321Z 4 080-099 42 42 42 65 65 65 F F F
62752023-09-22T23:26:35.324Z 5 100-119 44 44 44 68 68 68 F F F
62762023-09-22T23:26:35.328Z 6 120-139 44 44 44 68 68 68 F F F
62772023-09-22T23:26:35.331Z 7 140-159 47 47 47 71 71 71 F F F
62782023-09-22T23:26:35.334Z 8 160-179 44 44 44 68 68 68 F F F
62792023-09-22T23:26:35.337Z 9 180-199 46 46 46 70 70 70 F F F
62802023-09-22T23:26:35.340Z 10 200-219 46 46 46 70 70 70 F F F
62812023-09-22T23:26:35.343Z 11 220-239 35 35 35 52 52 52 F F F
62822023-09-22T23:26:35.346Z 12 240-259 42 42 42 64 64 64 F F F
62832023-09-22T23:26:35.349Z 13 260-279 45 45 45 69 69 69 F F F
62842023-09-22T23:26:35.352Z 14 280-299 45 45 45 69 69 69 F F F
62852023-09-22T23:26:35.355Z 15 300-319 46 46 46 70 70 70 F F F
62862023-09-22T23:26:35.358Z 16 320-339 47 47 47 71 71 71 F F F
62872023-09-22T23:26:35.361Z 17 340-359 47 47 47 71 71 71 F F F
62882023-09-22T23:26:35.364Z 18 360-379 45 45 45 69 69 69 F F F
62892023-09-22T23:26:35.366Z 19 380-399 46 46 46 70 70 70 F F F
62902023-09-22T23:26:35.370Z 20 400-419 46 46 46 70 70 70 F F F
62912023-09-22T23:26:35.373Z 21 420-439 46 46 46 70 70 70 F F F
62922023-09-22T23:26:35.376Z 22 440-459 43 43 43 67 67 67 F F F
62932023-09-22T23:26:35.379Z 23 460-479 47 47 47 71 71 71 F F F
62942023-09-22T23:26:35.382Z 24 480-499 46 46 46 70 70 70 F F F
62952023-09-22T23:26:35.385Z 25 500-519 47 47 47 71 71 71 F F F
62962023-09-22T23:26:35.388Z 26 520-539 47 47 47 71 71 71 F F F
62972023-09-22T23:26:35.390Z 27 540-559 45 45 45 69 69 69 F F F
62982023-09-22T23:26:35.394Z 28 560-579 45 45 45 69 69 69 F F F
62992023-09-22T23:26:35.397Z 29 580-599 46 46 46 70 70 70 F F F
63002023-09-22T23:26:35.400ZMax gen: 47, Max flush: 71
63012023-09-22T23:26:35.403Z
63022023-09-22T23:26:35.406ZBegin loop 24
63032023-09-22T23:26:35.409ZDownstairs to restart: 2
63042023-09-22T23:26:35.412Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.34498341Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63052023-09-22T23:26:35.415Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.345499089Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63062023-09-22T23:26:35.418Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.345582843Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63072023-09-22T23:26:35.421Z{"msg":"Crucible stats registered with UUID: 2c114d8f-6437-4dcc-8645-863e753ccba1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.345653667Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63082023-09-22T23:26:35.424Z{"msg":"Crucible 2c114d8f-6437-4dcc-8645-863e753ccba1 has session id: 2271dce6-3880-476f-9f1b-fd0b26a17797","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.345696727Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63092023-09-22T23:26:35.427ZCrucible runtime is spawned
63102023-09-22T23:26:35.430ZThe guest has requested activation with gen:48
63112023-09-22T23:26:35.433Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.345912858Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354,"task":"up_listen"}
63122023-09-22T23:26:35.436Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.345999166Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63132023-09-22T23:26:35.439Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.34606422Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63142023-09-22T23:26:35.442Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.346130591Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354,"looper":"1"}
63152023-09-22T23:26:35.445Z{"msg":"[1] 2c114d8f-6437-4dcc-8645-863e753ccba1 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.346325144Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354,"looper":"1"}
63162023-09-22T23:26:35.448Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.346396912Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354,"looper":"0"}
63172023-09-22T23:26:35.451Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.346475701Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354,"looper":"2"}
63182023-09-22T23:26:35.454Z{"msg":"[0] 2c114d8f-6437-4dcc-8645-863e753ccba1 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.346577253Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354,"looper":"0"}
63192023-09-22T23:26:35.457Z{"msg":"2c114d8f-6437-4dcc-8645-863e753ccba1 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.346952925Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63202023-09-22T23:26:35.460Z{"msg":"Set desired generation to :48","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.347011726Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63212023-09-22T23:26:35.463Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.347063553Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63222023-09-22T23:26:35.466Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.347408873Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63232023-09-22T23:26:35.469Z{"msg":"[1] 2c114d8f-6437-4dcc-8645-863e753ccba1 (07f1c1eb-58b6-4058-97ce-0a9460295176) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.347872124Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63242023-09-22T23:26:35.472Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.347947333Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63252023-09-22T23:26:35.475Z{"msg":"[1] client is_active_req TRUE, promote! session 07f1c1eb-58b6-4058-97ce-0a9460295176","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.347987473Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63262023-09-22T23:26:35.481Z{"msg":"[0] 2c114d8f-6437-4dcc-8645-863e753ccba1 (07f1c1eb-58b6-4058-97ce-0a9460295176) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.348066574Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63272023-09-22T23:26:35.485Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.348123641Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63282023-09-22T23:26:35.487Z{"msg":"[0] client is_active_req TRUE, promote! session 07f1c1eb-58b6-4058-97ce-0a9460295176","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.348259564Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63292023-09-22T23:26:35.490Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.348410313Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63302023-09-22T23:26:35.493Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.34855115Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63312023-09-22T23:26:35.496Z{"msg":"2c114d8f-6437-4dcc-8645-863e753ccba1 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.348638126Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63322023-09-22T23:26:35.499Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.348717553Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63332023-09-22T23:26:35.502Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.348798916Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63342023-09-22T23:26:35.504Z{"msg":"2c114d8f-6437-4dcc-8645-863e753ccba1 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.348873074Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63352023-09-22T23:26:35.508Z{"msg":"[1] 2c114d8f-6437-4dcc-8645-863e753ccba1 (07f1c1eb-58b6-4058-97ce-0a9460295176) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.351479073Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63362023-09-22T23:26:35.510Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.351561246Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63372023-09-22T23:26:35.513Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:35.351631802Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63382023-09-22T23:26:35.516Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.351705068Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63392023-09-22T23:26:35.519Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.351776657Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63402023-09-22T23:26:35.522Z{"msg":"2c114d8f-6437-4dcc-8645-863e753ccba1 WaitActive WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.351851687Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63412023-09-22T23:26:35.525Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.351921963Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63422023-09-22T23:26:35.529Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.351996467Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63432023-09-22T23:26:35.532Z{"msg":"[0] 2c114d8f-6437-4dcc-8645-863e753ccba1 (07f1c1eb-58b6-4058-97ce-0a9460295176) WaitActive WaitQuorum New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.352061121Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63442023-09-22T23:26:35.535Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.352105474Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63452023-09-22T23:26:35.537Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:35.35214945Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63462023-09-22T23:26:35.540Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.352187075Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63472023-09-22T23:26:35.543Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.352238941Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63482023-09-22T23:26:35.546Z{"msg":"2c114d8f-6437-4dcc-8645-863e753ccba1 WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.352281439Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63492023-09-22T23:26:35.549Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.352319755Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63502023-09-22T23:26:35.552Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:35.352357019Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63512023-09-22T23:26:36.349Z{"msg":"[2] 2c114d8f-6437-4dcc-8645-863e753ccba1 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.348050396Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354,"looper":"2"}
63522023-09-22T23:26:36.352Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.348145409Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63532023-09-22T23:26:36.355Z{"msg":"[2] 2c114d8f-6437-4dcc-8645-863e753ccba1 (07f1c1eb-58b6-4058-97ce-0a9460295176) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.349248262Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63542023-09-22T23:26:36.359Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.349301202Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63552023-09-22T23:26:36.362Z{"msg":"[2] client is_active_req TRUE, promote! session 07f1c1eb-58b6-4058-97ce-0a9460295176","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.349325445Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63562023-09-22T23:26:36.365Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.349944646Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63572023-09-22T23:26:36.369Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.350000474Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63582023-09-22T23:26:36.372Z{"msg":"2c114d8f-6437-4dcc-8645-863e753ccba1 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.350042816Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63592023-09-22T23:26:36.375Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.352521638Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63602023-09-22T23:26:36.378Z{"msg":"[2] 2c114d8f-6437-4dcc-8645-863e753ccba1 (07f1c1eb-58b6-4058-97ce-0a9460295176) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.352676988Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63612023-09-22T23:26:36.381Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.352740835Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63622023-09-22T23:26:36.384Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:36.352784148Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63632023-09-22T23:26:36.387Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.352822873Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63642023-09-22T23:26:36.390Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.352861629Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63652023-09-22T23:26:36.393Z{"msg":"2c114d8f-6437-4dcc-8645-863e753ccba1 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.352900928Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63662023-09-22T23:26:36.396Z{"msg":"[0]R flush_numbers[0..12]: [67, 70, 58, 66, 65, 68, 68, 71, 68, 70, 70, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.352940604Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63672023-09-22T23:26:36.399Z{"msg":"[0]R generation[0..12]: [43, 46, 39, 42, 42, 44, 44, 47, 44, 46, 46, 35]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.352990727Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63682023-09-22T23:26:36.402Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.353030938Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63692023-09-22T23:26:36.405Z{"msg":"[1]R flush_numbers[0..12]: [67, 70, 58, 66, 65, 68, 68, 71, 68, 70, 70, 52]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:48
63702023-09-22T23:26:36.408Z,"time":"2023-09-22T23:26:36.353053208Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63712023-09-22T23:26:36.411ZWait for a query_work_queue command to finish before sending IO
63722023-09-22T23:26:36.414Z{"msg":"[1]R generation[0..12]: [43, 46, 39, 42, 42, 44, 44, 47, 44, 46, 46, 35]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.353077165Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63732023-09-22T23:26:36.422Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.353120459Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63742023-09-22T23:26:36.425Z{"msg":"[2]R flush_numbers[0..12]: [67, 70, 58, 66, 65, 68, 68, 71, 68, 70, 70, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.353170014Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63752023-09-22T23:26:36.428Z{"msg":"[2]R generation[0..12]: [43, 46, 39, 42, 42, 44, 44, 47, 44, 46, 46, 35]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.353233577Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63762023-09-22T23:26:36.431Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.353312606Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63772023-09-22T23:26:36.434Z{"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.353361393Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63782023-09-22T23:26:36.438Z{"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.353407855Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63792023-09-22T23:26:36.440Z{"msg":"Next flush: 72","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.353463888Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63802023-09-22T23:26:36.443Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.353504875Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63812023-09-22T23:26:36.447Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.353543297Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63822023-09-22T23:26:36.454Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.353567823Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63832023-09-22T23:26:36.458Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.353625338Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63842023-09-22T23:26:36.462Z{"msg":"2c114d8f-6437-4dcc-8645-863e753ccba1 is now active with session: 07f1c1eb-58b6-4058-97ce-0a9460295176","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.353669539Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63852023-09-22T23:26:36.466Z{"msg":"2c114d8f-6437-4dcc-8645-863e753ccba1 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.353708948Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63862023-09-22T23:26:36.469Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.353730573Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63872023-09-22T23:26:36.472Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.353766423Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63882023-09-22T23:26:36.477Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.353819551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63892023-09-22T23:26:36.481Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.353882846Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63902023-09-22T23:26:36.484Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.353948092Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63912023-09-22T23:26:36.487Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.354014464Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63922023-09-22T23:26:36.491Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.354083275Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63932023-09-22T23:26:36.494Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.354201442Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63942023-09-22T23:26:36.497Z{"msg":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
63952023-09-22T23:26:36.500Z"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.354266433Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63962023-09-22T23:26:36.503Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.354310273Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63972023-09-22T23:26:36.506Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.354351281Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1354}
63982023-09-22T23:26:36.509ZLoading write count information from file "/tmp/test_repair_verify.data"
63992023-09-22T23:26:36.512ZRun Repair workload
64002023-09-22T23:26:36.515Z01/30 Write block 513 len 24576 data: 6 7 6 6 6 6
64012023-09-22T23:26:36.518Z02/30 Read block 36 len 28672
64022023-09-22T23:26:36.521Z03/30 Write block 447 len 28672 data: 4 6 7 7 7 7 6
64032023-09-22T23:26:36.524Z04/30 Write block 331 len 40960 data: 2 2 2 4 5 5 5 5 5 3
64042023-09-22T23:26:36.527Z05/30 Flush
64052023-09-22T23:26:36.530Z06/30 Write block 133 len 20480 data: 8 8 7 8 7
64062023-09-22T23:26:36.533Z07/30 Read block 154 len 4096
64072023-09-22T23:26:36.536Z08/30 Read block 393 len 4096
64082023-09-22T23:26:36.540Z09/30 Write block 564 len 28672 data: 5 4 4 7 7 7 4
64092023-09-22T23:26:36.543Z10/30 Write block 274 len 40960 data: 7 7 9 6 9 9 8 7 7 7
64102023-09-22T23:26:36.546Z11/30 Read block 62 len 8192
64112023-09-22T23:26:36.549Z12/30 Read block 279 len 12288
64122023-09-22T23:26:36.552Z13/30 Write block 548 len 16384 data: 6 6 6 6
64132023-09-22T23:26:36.555Z14/30 Flush
64142023-09-22T23:26:36.558Z15/30 Write block 303 len 28672 data: 6 7 6 6 6 6 4
64152023-09-22T23:26:36.561Z16/30 Read block 557 len 32768
64162023-09-22T23:26:36.565Z17/30 Write block 181 len 16384 data: 3 4 2 2
64172023-09-22T23:26:36.569Z18/30 Read block 490 len 12288
64182023-09-22T23:26:36.573Z19/30 Read block 337 len 40960
64192023-09-22T23:26:36.575Z20/30 Read block 456 len 4096
64202023-09-22T23:26:36.578Z21/30 Write block 63 len 28672 data: 8 8 9 9 9 10 11
64212023-09-22T23:26:36.581Z22/30 Write block 419 len 36864 data: 4 5 5 4 3 4 2 3 5
64222023-09-22T23:26:36.584Z23/30 Write block 512 len 28672 data: 5 7 8 7 7 7 7
64232023-09-22T23:26:36.588Z24/30 Read block 201 len 16384
64242023-09-22T23:26:36.591Z25/30 Write block 157 len 36864 data: 2 5 7 7 7 6 5 5 4
64252023-09-22T23:26:36.594Z26/30 Read block 327 len 28672
64262023-09-22T23:26:36.597Z27/30 Read block 167 len 32768
64272023-09-22T23:26:36.600Z28/30 Write block 109 len 36864 data: 6 7 5 5 4 3 3 3 3
64282023-09-22T23:26:36.603Z29/30 Read block 220 len 12288
64292023-09-22T23:26:36.606Z30/30 Read block 530 len 40960
64302023-09-22T23:26:36.609Z----------------------------------------------------------------
64312023-09-22T23:26:36.612Z Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:30
64322023-09-22T23:26:36.615ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
64332023-09-22T23:26:36.618Z 1 Acked 1000 Write 6 Done Done Done false
64342023-09-22T23:26:36.621Z 2 Acked 1001 Read 7 Done Done Done false
64352023-09-22T23:26:36.627Z 3 Acked 1002 Write 7 Done Done Sent false
64362023-09-22T23:26:36.633Z 4 Acked 1003 Write 10 Done Done Sent false
64372023-09-22T23:26:36.636Z 5 Acked 1004 Flush 0 Done Done Sent false
64382023-09-22T23:26:36.640Z 6 Acked 1005 Write 5 Done Done Sent false
64392023-09-22T23:26:36.643Z 7 Acked 1006 Read 1 Done Done Sent false
64402023-09-22T23:26:36.646Z 8 Acked 1007 Read 1 Done Done Sent false
64412023-09-22T23:26:36.649Z 9 Acked 1008 Write 7 Done Done Sent false
64422023-09-22T23:26:36.653Z 10 Acked 1009 Write 10 Done Done Sent false
64432023-09-22T23:26:36.656Z 11 Acked 1010 Read 2 Done Done Sent false
64442023-09-22T23:26:36.659Z 12 Acked 1011 Read 3 Done Done Sent false
64452023-09-22T23:26:36.662Z 13 Acked 1012 Write 4 Done Done Sent false
64462023-09-22T23:26:36.665Z 14 Acked 1013 Flush 0 Done Done Sent false
64472023-09-22T23:26:36.668Z 15 Acked 1014 Write 7 Done Done Sent false
64482023-09-22T23:26:36.671Z 16 Acked 1015 Read 8 Done Done Sent false
64492023-09-22T23:26:36.674Z 17 Acked 1016 Write 4 Done Done Sent false
64502023-09-22T23:26:36.677Z 18 Acked 1017 Read 3 Done Done Sent false
64512023-09-22T23:26:36.681Z 19 Acked 1018 Read 10 Done Done Sent false
64522023-09-22T23:26:36.684Z 20 Acked 1019 Read 1 Done Done Sent false
64532023-09-22T23:26:36.687Z 21 Acked 1020 Write 7 Done Done Sent false
64542023-09-22T23:26:36.691Z 22 Acked 1021 Write 9 Done Done Sent false
64552023-09-22T23:26:36.695Z 23 Acked 1022 Write 7 Done Done Sent false
64562023-09-22T23:26:36.698Z 24 Acked 1023 Read 4 Done Done Sent false
64572023-09-22T23:26:36.701Z 25 Acked 1024 Write 9 Done Done Sent false
64582023-09-22T23:26:36.704Z 26 Acked 1025 Read 7 Done Done Sent false
64592023-09-22T23:26:36.707Z 27 Acked 1026 Read 8 Done Done Sent false
64602023-09-22T23:26:36.710Z 28 Acked 1027 Write 9 Done Done Sent false
64612023-09-22T23:26:36.713Z 29 Acked 1028 Read 3 Done Done Sent false
64622023-09-22T23:26:36.716Z 30 Acked 1029 Read 10 Done Done Sent false
64632023-09-22T23:26:36.719Z STATES DS:0 DS:1 DS:2 TOTAL
64642023-09-22T23:26:36.722Z New 0 0 0 0
64652023-09-22T23:26:36.725Z Sent 0 0 28 28
64662023-09-22T23:26:36.729Z Done 30 30 2 62
64672023-09-22T23:26:36.732Z Skipped 0 0 0 0
64682023-09-22T23:26:36.735Z Error 0 0 0 0
64692023-09-22T23:26:36.738ZLast Flush: 1013 1013 0
64702023-09-22T23:26:36.741ZDownstairs last five completed:
64712023-09-22T23:26:36.744ZUpstairs last five completed: 30 29 28 27 26 25
64722023-09-22T23:26:36.747ZWrote out file "/tmp/test_repair_verify.data"
64732023-09-22T23:26:36.750Z
64742023-09-22T23:26:36.753ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
64752023-09-22T23:26:36.756Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.507252612Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1355}
64762023-09-22T23:26:36.759Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.507621251Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1355}
64772023-09-22T23:26:36.762Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.50764499Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1355}
64782023-09-22T23:26:36.765Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.540410109Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1355}
64792023-09-22T23:26:36.768Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.540442945Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1355}
64802023-09-22T23:26:36.771Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.540462057Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1355}
64812023-09-22T23:26:36.775Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.564643282Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1355}
64822023-09-22T23:26:36.778Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.564697427Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1355}
64832023-09-22T23:26:36.781Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:36.564718585Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1355}
64842023-09-22T23:26:36.783ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
64852023-09-22T23:26:36.786Z 0 000-019 43 43 43 67 67 67 F F F
64862023-09-22T23:26:36.789Z 1 020-039 46 46 46 70 70 70 F F F
64872023-09-22T23:26:36.792Z 2 040-059 39 39 39 58 58 58 F F F
64882023-09-22T23:26:36.795ZError: Difference in extent metadata found!
64892023-09-22T23:26:36.798Z 3 060-079 42 42 42 66 66 66 T T F <---
64902023-09-22T23:26:36.801Z 4 080-099 42 42 42 65 65 65 F F F
64912023-09-22T23:26:36.804Z 5 100-119 44 44 44 68 68 68 T T F <---
64922023-09-22T23:26:36.807Z 6 120-139 48 48 44 73 73 68 F F F <---
64932023-09-22T23:26:36.810Z 7 140-159 47 47 47 71 71 71 T T F <---
64942023-09-22T23:26:36.813Z 8 160-179 44 44 44 68 68 68 T T F <---
64952023-09-22T23:26:36.816Z 9 180-199 46 46 46 70 70 70 T T F <---
64962023-09-22T23:26:36.820Z 10 200-219 46 46 46 70 70 70 F F F
64972023-09-22T23:26:36.823Z 11 220-239 35 35 35 52 52 52 F F F
64982023-09-22T23:26:36.826Z 12 240-259 42 42 42 64 64 64 F F F
64992023-09-22T23:26:36.831Z 13 260-279 48 48 45 73 73 69 F F F <---
65002023-09-22T23:26:36.834Z 14 280-299 48 48 45 73 73 69 F F F <---
65012023-09-22T23:26:36.837Z 15 300-319 46 46 46 70 70 70 T T F <---
65022023-09-22T23:26:36.840Z 16 320-339 48 48 47 72 72 71 F F F <---
65032023-09-22T23:26:36.843Z 17 340-359 48 48 47 72 72 71 F F F <---
65042023-09-22T23:26:36.846Z 18 360-379 45 45 45 69 69 69 F F F
65052023-09-22T23:26:36.849Z 19 380-399 46 46 46 70 70 70 F F F
65062023-09-22T23:26:36.852Z 20 400-419 46 46 46 70 70 70 T T F <---
65072023-09-22T23:26:36.855Z 21 420-439 46 46 46 70 70 70 T T F <---
65082023-09-22T23:26:36.858Z 22 440-459 48 48 43 72 72 67 F F F <---
65092023-09-22T23:26:36.861Z 23 460-479 47 47 47 71 71 71 F F F
65102023-09-22T23:26:36.864Z 24 480-499 46 46 46 70 70 70 F F F
65112023-09-22T23:26:36.867Z 25 500-519 48 48 47 72 72 71 T T T <---
65122023-09-22T23:26:36.870Z 26 520-539 47 47 47 71 71 71 F F F
65132023-09-22T23:26:36.874Z 27 540-559 48 48 45 73 73 69 F F F <---
65142023-09-22T23:26:36.877Z 28 560-579 48 48 45 73 73 69 F F F <---
65152023-09-22T23:26:36.880Z 29 580-599 46 46 46 70 70 70 F F F
65162023-09-22T23:26:36.884ZMax gen: 48, Max flush: 73
65172023-09-22T23:26:36.895ZA Difference in extent metadata is expected here
65182023-09-22T23:26:36.898ZOn loop 24
65192023-09-22T23:26:36.901Z
65202023-09-22T23:26:36.904ZVerifying data now
65212023-09-22T23:26:43.339ZNo change after verify
65222023-09-22T23:26:43.343ZLoop: 24 Downstairs dump after verify (and repair):
65232023-09-22T23:26:43.357Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.355505267Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1360}
65242023-09-22T23:26:43.361Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.355894451Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1360}
65252023-09-22T23:26:43.364Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.355919254Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1360}
65262023-09-22T23:26:43.376Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.374801625Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1360}
65272023-09-22T23:26:43.381Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.374854167Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1360}
65282023-09-22T23:26:43.384Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.374874585Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1360}
65292023-09-22T23:26:43.388Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.385974269Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1360}
65302023-09-22T23:26:43.391Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.386025845Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1360}
65312023-09-22T23:26:43.394Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.386064121Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1360}
65322023-09-22T23:26:43.398ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
65332023-09-22T23:26:43.401Z 0 000-019 43 43 43 67 67 67 F F F
65342023-09-22T23:26:43.405Z 1 020-039 46 46 46 70 70 70 F F F
65352023-09-22T23:26:43.408Z 2 040-059 39 39 39 58 58 58 F F F
65362023-09-22T23:26:43.412Z 3 060-079 49 49 49 74 74 74 F F F
65372023-09-22T23:26:43.416Z 4 080-099 42 42 42 65 65 65 F F F
65382023-09-22T23:26:43.419Z 5 100-119 49 49 49 74 74 74 F F F
65392023-09-22T23:26:43.423Z 6 120-139 48 48 48 73 73 73 F F F
65402023-09-22T23:26:43.427Z 7 140-159 49 49 49 74 74 74 F F F
65412023-09-22T23:26:43.432Z 8 160-179 49 49 49 74 74 74 F F F
65422023-09-22T23:26:43.459Z 9 180-199 49 49 49 74 74 74 F F F
65432023-09-22T23:26:43.463Z 10 200-219 46 46 46 70 70 70 F F F
65442023-09-22T23:26:43.466Z 11 220-239 35 35 35 52 52 52 F F F
65452023-09-22T23:26:43.469Z 12 240-259 42 42 42 64 64 64 F F F
65462023-09-22T23:26:43.473Z 13 260-279 48 48 48 73 73 73 F F F
65472023-09-22T23:26:43.476Z 14 280-299 48 48 48 73 73 73 F F F
65482023-09-22T23:26:43.479Z 15 300-319 49 49 49 74 74 74 F F F
65492023-09-22T23:26:43.485Z 16 320-339 48 48 48 72 72 72 F F F
65502023-09-22T23:26:43.489Z 17 340-359 48 48 48 72 72 72 F F F
65512023-09-22T23:26:43.492Z 18 360-379 45 45 45 69 69 69 F F F
65522023-09-22T23:26:43.497Z 19 380-399 46 46 46 70 70 70 F F F
65532023-09-22T23:26:43.501Z 20 400-419 49 49 49 74 74 74 F F F
65542023-09-22T23:26:43.505Z 21 420-439 49 49 49 74 74 74 F F F
65552023-09-22T23:26:43.510Z 22 440-459 48 48 48 72 72 72 F F F
65562023-09-22T23:26:43.514Z 23 460-479 47 47 47 71 71 71 F F F
65572023-09-22T23:26:43.517Z 24 480-499 46 46 46 70 70 70 F F F
65582023-09-22T23:26:43.520Z 25 500-519 49 49 49 74 74 74 F F F
65592023-09-22T23:26:43.524Z 26 520-539 47 47 47 71 71 71 F F F
65602023-09-22T23:26:43.527Z 27 540-559 48 48 48 73 73 73 F F F
65612023-09-22T23:26:43.530Z 28 560-579 48 48 48 73 73 73 F F F
65622023-09-22T23:26:43.534Z 29 580-599 46 46 46 70 70 70 F F F
65632023-09-22T23:26:43.538ZMax gen: 49, Max flush: 74
65642023-09-22T23:26:43.542Z
65652023-09-22T23:26:43.545ZBegin loop 25
65662023-09-22T23:26:43.548ZDownstairs to restart: 0
65672023-09-22T23:26:43.552Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.433573932Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
65682023-09-22T23:26:43.567Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.433942941Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
65692023-09-22T23:26:43.570Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.43400309Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
65702023-09-22T23:26:43.574Z{"msg":"Crucible stats registered with UUID: f38ba227-3d9a-4ae7-9e39-e54642cdea20","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.434029384Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
65712023-09-22T23:26:43.578Z{"msg":"Crucible f38ba227-3d9a-4ae7-9e39-e54642cdea20 has session id: 400bcbaa-bf79-4d17-88ec-4ec39f18d30d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.43407556Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
65722023-09-22T23:26:43.582ZCrucible runtime is spawned
65732023-09-22T23:26:43.586Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.434319109Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362,"The guest has requested activation with gen:50
65742023-09-22T23:26:43.589Ztask":"up_listen"}
65752023-09-22T23:26:43.593Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.434449391Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
65762023-09-22T23:26:43.596Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.43449789Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
65772023-09-22T23:26:43.599Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.434537779Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362,"looper":"1"}
65782023-09-22T23:26:43.603Z{"msg":"[1] f38ba227-3d9a-4ae7-9e39-e54642cdea20 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.434801401Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362,"looper":"1"}
65792023-09-22T23:26:43.607Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.434941876Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362,"looper":"2"}
65802023-09-22T23:26:43.611Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.43522975Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362,"looper":"0"}
65812023-09-22T23:26:43.616Z{"msg":"[2] f38ba227-3d9a-4ae7-9e39-e54642cdea20 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.435271619Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362,"looper":"2"}
65822023-09-22T23:26:43.619Z{"msg":"f38ba227-3d9a-4ae7-9e39-e54642cdea20 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.435743046Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
65832023-09-22T23:26:43.623Z{"msg":"Set desired generation to :50","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.435807824Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
65842023-09-22T23:26:43.626Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.435926252Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
65852023-09-22T23:26:43.629Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.43627286Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
65862023-09-22T23:26:43.632Z{"msg":"[1] f38ba227-3d9a-4ae7-9e39-e54642cdea20 (e4f8579a-f73c-4928-9070-10d84f0e46ce) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.436628776Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
65872023-09-22T23:26:43.636Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.436692492Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
65882023-09-22T23:26:43.639Z{"msg":"[1] client is_active_req TRUE, promote! session e4f8579a-f73c-4928-9070-10d84f0e46ce","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.436777705Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
65892023-09-22T23:26:43.642Z{"msg":"[2] f38ba227-3d9a-4ae7-9e39-e54642cdea20 (e4f8579a-f73c-4928-9070-10d84f0e46ce) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.437037206Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
65902023-09-22T23:26:43.645Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.437116538Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
65912023-09-22T23:26:43.650Z{"msg":"[2] client is_active_req TRUE, promote! session e4f8579a-f73c-4928-9070-10d84f0e46ce","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.437185665Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
65922023-09-22T23:26:43.653Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.437451214Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
65932023-09-22T23:26:43.657Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.437513905Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
65942023-09-22T23:26:43.663Z{"msg":"f38ba227-3d9a-4ae7-9e39-e54642cdea20 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.437557449Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
65952023-09-22T23:26:43.667Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.437701259Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
65962023-09-22T23:26:43.670Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.437771853Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
65972023-09-22T23:26:43.674Z{"msg":"f38ba227-3d9a-4ae7-9e39-e54642cdea20 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.437843666Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
65982023-09-22T23:26:43.679Z{"msg":"[1] f38ba227-3d9a-4ae7-9e39-e54642cdea20 (e4f8579a-f73c-4928-9070-10d84f0e46ce) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.439835417Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
65992023-09-22T23:26:43.682Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.440028306Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66002023-09-22T23:26:43.685Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:43.440112094Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66012023-09-22T23:26:43.689Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.440184011Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66022023-09-22T23:26:43.692Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.440250637Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66032023-09-22T23:26:43.695Z{"msg":"f38ba227-3d9a-4ae7-9e39-e54642cdea20 New WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.440318331Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66042023-09-22T23:26:43.699Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.440390961Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66052023-09-22T23:26:43.702Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.440458856Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66062023-09-22T23:26:43.705Z{"msg":"[2] f38ba227-3d9a-4ae7-9e39-e54642cdea20 (e4f8579a-f73c-4928-9070-10d84f0e46ce) New WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.440534415Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66072023-09-22T23:26:43.708Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.440607353Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66082023-09-22T23:26:43.711Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:43.440680382Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66092023-09-22T23:26:43.714Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.440721181Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66102023-09-22T23:26:43.717Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.440756754Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66112023-09-22T23:26:43.720Z{"msg":"f38ba227-3d9a-4ae7-9e39-e54642cdea20 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.440790991Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66122023-09-22T23:26:43.723Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.440825545Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66132023-09-22T23:26:43.726Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:43.440857834Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66142023-09-22T23:26:44.439Z{"msg":"[0] f38ba227-3d9a-4ae7-9e39-e54642cdea20 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.437220093Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362,"looper":"0"}
66152023-09-22T23:26:44.442Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.437308609Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66162023-09-22T23:26:44.446Z{"msg":"[0] f38ba227-3d9a-4ae7-9e39-e54642cdea20 (e4f8579a-f73c-4928-9070-10d84f0e46ce) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.438234291Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66172023-09-22T23:26:44.449Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.43828657Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66182023-09-22T23:26:44.452Z{"msg":"[0] client is_active_req TRUE, promote! session e4f8579a-f73c-4928-9070-10d84f0e46ce","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.438331132Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66192023-09-22T23:26:44.456Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.438908144Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66202023-09-22T23:26:44.480Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.438966545Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66212023-09-22T23:26:44.486Z{"msg":"f38ba227-3d9a-4ae7-9e39-e54642cdea20 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.439010154Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66222023-09-22T23:26:44.489Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.441316051Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66232023-09-22T23:26:44.493Z{"msg":"[0] f38ba227-3d9a-4ae7-9e39-e54642cdea20 (e4f8579a-f73c-4928-9070-10d84f0e46ce) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.441439614Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66242023-09-22T23:26:44.495Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.441556722Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66252023-09-22T23:26:44.499Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:44.441616994Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66262023-09-22T23:26:44.502Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.441637942Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66272023-09-22T23:26:44.505Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.441688302Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66282023-09-22T23:26:44.508Z{"msg":"f38ba227-3d9a-4ae7-9e39-e54642cdea20 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.441728474Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66292023-09-22T23:26:44.511Z{"msg":"[0]R flush_numbers[0..12]: [67, 70, 58, 74, 65, 74, 73, 74, 74, 74, 70, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.441751658Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66302023-09-22T23:26:44.514Z{"msg":"[0]R generation[0..12]: [43, 46, 39, 49, 42, 49, 48, 49, 49, 49, 46, 35]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.441794618Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66312023-09-22T23:26:44.517Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:50
66322023-09-22T23:26:44.520ZWait for a query_work_queue command to finish before sending IO
66332023-09-22T23:26:44.523Z,"time":"2023-09-22T23:26:44.441818298Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66342023-09-22T23:26:44.526Z{"msg":"[1]R flush_numbers[0..12]: [67, 70, 58, 74, 65, 74, 73, 74, 74, 74, 70, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.442038676Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66352023-09-22T23:26:44.529Z{"msg":"[1]R generation[0..12]: [43, 46, 39, 49, 42, 49, 48, 49, 49, 49, 46, 35]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.442113918Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66362023-09-22T23:26:44.535Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.44221214Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66372023-09-22T23:26:44.538Z{"msg":"[2]R flush_numbers[0..12]: [67, 70, 58, 74, 65, 74, 73, 74, 74, 74, 70, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.442261642Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66382023-09-22T23:26:44.541Z{"msg":"[2]R generation[0..12]: [43, 46, 39, 49, 42, 49, 48, 49, 49, 49, 46, 35]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.442302334Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66392023-09-22T23:26:44.546Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.44235034Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66402023-09-22T23:26:44.550Z{"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.4423887Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66412023-09-22T23:26:44.553Z{"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.442438068Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66422023-09-22T23:26:44.556Z{"msg":"Next flush: 75","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.442481976Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66432023-09-22T23:26:44.560Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.442529132Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66442023-09-22T23:26:44.563Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.442567598Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66452023-09-22T23:26:44.569Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.442613549Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66462023-09-22T23:26:44.573Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.44266566Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66472023-09-22T23:26:44.576Z{"msg":"f38ba227-3d9a-4ae7-9e39-e54642cdea20 is now active with session: e4f8579a-f73c-4928-9070-10d84f0e46ce","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.442705875Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66482023-09-22T23:26:44.579Z{"msg":"f38ba227-3d9a-4ae7-9e39-e54642cdea20 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.442745547Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66492023-09-22T23:26:44.582Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.442790582Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66502023-09-22T23:26:44.589Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.442832951Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66512023-09-22T23:26:44.592Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.442871356Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66522023-09-22T23:26:44.595Z{"msg":"[0] received reconcile message","v":0,"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
66532023-09-22T23:26:44.599Zname":"crucible","level":30,"time":"2023-09-22T23:26:44.4429152Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66542023-09-22T23:26:44.604Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.442953332Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66552023-09-22T23:26:44.607Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.442990917Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66562023-09-22T23:26:44.610Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.443028634Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66572023-09-22T23:26:44.613Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.443066277Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66582023-09-22T23:26:44.616Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.443103697Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66592023-09-22T23:26:44.621Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.443141422Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66602023-09-22T23:26:44.624Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.44317883Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1362}
66612023-09-22T23:26:44.627ZLoading write count information from file "/tmp/test_repair_verify.data"
66622023-09-22T23:26:44.629ZRun Repair workload
66632023-09-22T23:26:44.632Z01/30 Write block 549 len 20480 data: 7 7 7 6 6
66642023-09-22T23:26:44.635Z02/30 Write block 577 len 12288 data: 3 3 3
66652023-09-22T23:26:44.638Z03/30 Read block 342 len 24576
66662023-09-22T23:26:44.641Z04/30 Write block 198 len 4096 data: 4
66672023-09-22T23:26:44.644Z05/30 Write block 440 len 8192 data: 2 4
66682023-09-22T23:26:44.648Z06/30 Read block 347 len 36864
66692023-09-22T23:26:44.651Z07/30 Read block 256 len 28672
66702023-09-22T23:26:44.654Z08/30 Read block 452 len 32768
66712023-09-22T23:26:44.657Z09/30 Read block 238 len 20480
66722023-09-22T23:26:44.660Z10/30 Flush
66732023-09-22T23:26:44.663Z11/30 Write block 576 len 12288 data: 3 4 4
66742023-09-22T23:26:44.667Z12/30 Read block 186 len 24576
66752023-09-22T23:26:44.670Z13/30 Read block 381 len 4096
66762023-09-22T23:26:44.673Z14/30 Write block 398 len 40960 data: 6 6 7 6 8 7 7 7 6 6
66772023-09-22T23:26:44.676Z15/30 Read block 231 len 20480
66782023-09-22T23:26:44.679Z16/30 Read block 393 len 24576
66792023-09-22T23:26:44.683Z17/30 Read block 189 len 4096
66802023-09-22T23:26:44.686Z18/30 Read block 201 len 4096
66812023-09-22T23:26:44.689Z19/30 Write block 271 len 24576 data: 9 9 8 8 8 10
66822023-09-22T23:26:44.692Z20/30 Write block 323 len 4096 data: 7
66832023-09-22T23:26:44.695Z21/30 Read block 349 len 16384
66842023-09-22T23:26:44.698Z22/30 Write block 598 len 4096 data: 3
66852023-09-22T23:26:44.701Z23/30 Read block 557 len 28672
66862023-09-22T23:26:44.704Z24/30 Read block 254 len 40960
66872023-09-22T23:26:44.707Z25/30 Read block 215 len 32768
66882023-09-22T23:26:44.710Z26/30 Write block 31 len 32768 data: 3 2 2 4 5 8 8 8
66892023-09-22T23:26:44.713Z27/30 Write block 394 len 16384 data: 4 5 5 5
66902023-09-22T23:26:44.716Z28/30 Read block 78 len 24576
66912023-09-22T23:26:44.719Z29/30 Write block 588 len 20480 data: 3 3 3 3 5
66922023-09-22T23:26:44.722Z30/30 Read block 350 len 12288
66932023-09-22T23:26:44.725Z----------------------------------------------------------------
66942023-09-22T23:26:44.728Z Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:30
66952023-09-22T23:26:44.731ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
66962023-09-22T23:26:44.733Z 1 Acked 1000 Write 5 Done Done Done false
66972023-09-22T23:26:44.736Z 2 Acked 1001 Write 3 Done Done Done false
66982023-09-22T23:26:44.739Z 3 Acked 1002 Read 6 Done Done Done false
66992023-09-22T23:26:44.742Z 4 Acked 1003 Write 1 Sent Done Done false
67002023-09-22T23:26:44.745Z 5 Acked 1004 Write 2 Sent Done Done false
67012023-09-22T23:26:44.748Z 6 Acked 1005 Read 9 Sent Done Done false
67022023-09-22T23:26:44.751Z 7 Acked 1006 Read 7 Sent Done Done false
67032023-09-22T23:26:44.754Z 8 Acked 1007 Read 8 Sent Done Done false
67042023-09-22T23:26:44.756Z 9 Acked 1008 Read 5 Sent Done Done false
67052023-09-22T23:26:44.759Z 10 Acked 1009 Flush 0 Sent Done Done false
67062023-09-22T23:26:44.762Z 11 Acked 1010 Write 3 Sent Done Done false
67072023-09-22T23:26:44.766Z 12 Acked 1011 Read 6 Sent Done Done false
67082023-09-22T23:26:44.769Z 13 Acked 1012 Read 1 Sent Done Done false
67092023-09-22T23:26:44.772Z 14 Acked 1013 Write 10 Sent Done Done false
67102023-09-22T23:26:44.775Z 15 Acked 1014 Read 5 Sent Done Done false
67112023-09-22T23:26:44.778Z 16 Acked 1015 Read 6 Sent Done Done false
67122023-09-22T23:26:44.781Z 17 Acked 1016 Read 1 Sent Done Done false
67132023-09-22T23:26:44.784Z 18 Acked 1017 Read 1 Sent Done Done false
67142023-09-22T23:26:44.787Z 19 Acked 1018 Write 6 Sent Done Done false
67152023-09-22T23:26:44.790Z 20 Acked 1019 Write 1 Sent Done Done false
67162023-09-22T23:26:44.793Z 21 Acked 1020 Read 4 Sent Done Done false
67172023-09-22T23:26:44.795Z 22 Acked 1021 Write 1 Sent Done Done false
67182023-09-22T23:26:44.799Z 23 Acked 1022 Read 7 Sent Done Done false
67192023-09-22T23:26:44.802Z 24 Acked 1023 Read 10 Sent Done Done false
67202023-09-22T23:26:44.805Z 25 Acked 1024 Read 8 Sent Done Done false
67212023-09-22T23:26:44.808Z 26 Acked 1025 Write 8 Sent Done Done false
67222023-09-22T23:26:44.812Z 27 Acked 1026 Write 4 Sent Done Done false
67232023-09-22T23:26:44.815Z 28 Acked 1027 Read 6 Sent Done Done false
67242023-09-22T23:26:44.817Z 29 Acked 1028 Write 5 Sent Sent Done false
67252023-09-22T23:26:44.821Z 30 Acked 1029 Read 3 Sent Sent Done false
67262023-09-22T23:26:44.824Z STATES DS:0 DS:1 DS:2 TOTAL
67272023-09-22T23:26:44.827Z New 0 0 0 0
67282023-09-22T23:26:44.830Z Sent 27 2 0 29
67292023-09-22T23:26:44.833Z Done 3 28 30 61
67302023-09-22T23:26:44.835Z Skipped 0 0 0 0
67312023-09-22T23:26:44.838Z Error 0 0 0 0
67322023-09-22T23:26:44.842ZLast Flush: 0 1009 1009
67332023-09-22T23:26:44.845ZError: Difference in extent metadata found!
67342023-09-22T23:26:44.848ZDownstairs last five completed:
67352023-09-22T23:26:44.851ZUpstairs last five completed: 30 29 28 27 26 25
67362023-09-22T23:26:44.854ZWrote out file "/tmp/test_repair_verify.data"
67372023-09-22T23:26:44.856Z
67382023-09-22T23:26:44.859ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
67392023-09-22T23:26:44.863Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.57100772Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1363}
67402023-09-22T23:26:44.865Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.571311391Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1363}
67412023-09-22T23:26:44.868Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.571335539Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1363}
67422023-09-22T23:26:44.871Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.602872373Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1363}
67432023-09-22T23:26:44.874Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.602907879Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1363}
67442023-09-22T23:26:44.877Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.60292709Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1363}
67452023-09-22T23:26:44.881Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.624169531Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1363}
67462023-09-22T23:26:44.884Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.624201361Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1363}
67472023-09-22T23:26:44.887Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:44.624220767Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1363}
67482023-09-22T23:26:44.890ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
67492023-09-22T23:26:44.893Z 0 000-019 43 43 43 67 67 67 F F F
67502023-09-22T23:26:44.897Z 1 020-039 46 46 46 70 70 70 F T T <---
67512023-09-22T23:26:44.900Z 2 040-059 39 39 39 58 58 58 F F F
67522023-09-22T23:26:44.903Z 3 060-079 49 49 49 74 74 74 F F F
67532023-09-22T23:26:44.906Z 4 080-099 42 42 42 65 65 65 F F F
67542023-09-22T23:26:44.909Z 5 100-119 49 49 49 74 74 74 F F F
67552023-09-22T23:26:44.912Z 6 120-139 48 48 48 73 73 73 F F F
67562023-09-22T23:26:44.915Z 7 140-159 49 49 49 74 74 74 F F F
67572023-09-22T23:26:44.918Z 8 160-179 49 49 49 74 74 74 F F F
67582023-09-22T23:26:44.921Z 9 180-199 49 50 50 74 75 75 F F F <---
67592023-09-22T23:26:44.924Z 10 200-219 46 46 46 70 70 70 F F F
67602023-09-22T23:26:44.927Z 11 220-239 35 35 35 52 52 52 F F F
67612023-09-22T23:26:44.930Z 12 240-259 42 42 42 64 64 64 F F F
67622023-09-22T23:26:44.933Z 13 260-279 48 48 48 73 73 73 F T T <---
67632023-09-22T23:26:44.936Z 14 280-299 48 48 48 73 73 73 F F F
67642023-09-22T23:26:44.938Z 15 300-319 49 49 49 74 74 74 F F F
67652023-09-22T23:26:44.941Z 16 320-339 48 48 48 72 72 72 F T T <---
67662023-09-22T23:26:44.945Z 17 340-359 48 48 48 72 72 72 F F F
67672023-09-22T23:26:44.948Z 18 360-379 45 45 45 69 69 69 F F F
67682023-09-22T23:26:44.951Z 19 380-399 46 46 46 70 70 70 F T T <---
67692023-09-22T23:26:44.954Z 20 400-419 49 49 49 74 74 74 F T T <---
67702023-09-22T23:26:44.957Z 21 420-439 49 49 49 74 74 74 F F F
67712023-09-22T23:26:44.960Z 22 440-459 48 50 50 72 75 75 F F F <---
67722023-09-22T23:26:44.963Z 23 460-479 47 47 47 71 71 71 F F F
67732023-09-22T23:26:44.966Z 24 480-499 46 46 46 70 70 70 F F F
67742023-09-22T23:26:44.969Z 25 500-519 49 49 49 74 74 74 F F F
67752023-09-22T23:26:44.971Z 26 520-539 47 47 47 71 71 71 F F F
67762023-09-22T23:26:44.974Z 27 540-559 48 50 50 73 75 75 T F F <---
67772023-09-22T23:26:44.977Z 28 560-579 48 50 50 73 75 75 T T T <---
67782023-09-22T23:26:44.980Z 29 580-599 46 46 46 70 70 70 F T T <---
67792023-09-22T23:26:44.983ZMax gen: 50, Max flush: 75
67802023-09-22T23:26:44.986ZA Difference in extent metadata is expected here
67812023-09-22T23:26:44.989ZOn loop 25
67822023-09-22T23:26:44.992Z
67832023-09-22T23:26:44.995ZVerifying data now
67842023-09-22T23:26:50.823ZNo change after verify
67852023-09-22T23:26:50.838ZLoop: 25 Downstairs dump after verify (and repair):
67862023-09-22T23:26:50.845Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.840578522Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1368}
67872023-09-22T23:26:50.848Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.840967928Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1368}
67882023-09-22T23:26:50.851Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.841011435Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1368}
67892023-09-22T23:26:50.861Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.860193716Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1368}
67902023-09-22T23:26:50.864Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.860244594Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1368}
67912023-09-22T23:26:50.868Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.860265025Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1368}
67922023-09-22T23:26:50.872Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.871326961Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1368}
67932023-09-22T23:26:50.875Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.871378429Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1368}
67942023-09-22T23:26:50.879Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.87139895Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1368}
67952023-09-22T23:26:50.885ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
67962023-09-22T23:26:50.888Z 0 000-019 43 43 43 67 67 67 F F F
67972023-09-22T23:26:50.891Z 1 020-039 51 51 51 76 76 76 F F F
67982023-09-22T23:26:50.894Z 2 040-059 39 39 39 58 58 58 F F F
67992023-09-22T23:26:50.897Z 3 060-079 49 49 49 74 74 74 F F F
68002023-09-22T23:26:50.900Z 4 080-099 42 42 42 65 65 65 F F F
68012023-09-22T23:26:50.903Z 5 100-119 49 49 49 74 74 74 F F F
68022023-09-22T23:26:50.906Z 6 120-139 48 48 48 73 73 73 F F F
68032023-09-22T23:26:50.909Z 7 140-159 49 49 49 74 74 74 F F F
68042023-09-22T23:26:50.913Z 8 160-179 49 49 49 74 74 74 F F F
68052023-09-22T23:26:50.916Z 9 180-199 50 50 50 75 75 75 F F F
68062023-09-22T23:26:50.919Z 10 200-219 46 46 46 70 70 70 F F F
68072023-09-22T23:26:50.922Z 11 220-239 35 35 35 52 52 52 F F F
68082023-09-22T23:26:50.925Z 12 240-259 42 42 42 64 64 64 F F F
68092023-09-22T23:26:50.929Z 13 260-279 51 51 51 76 76 76 F F F
68102023-09-22T23:26:50.932Z 14 280-299 48 48 48 73 73 73 F F F
68112023-09-22T23:26:50.935Z 15 300-319 49 49 49 74 74 74 F F F
68122023-09-22T23:26:50.938Z 16 320-339 51 51 51 76 76 76 F F F
68132023-09-22T23:26:50.941Z 17 340-359 48 48 48 72 72 72 F F F
68142023-09-22T23:26:50.944Z 18 360-379 45 45 45 69 69 69 F F F
68152023-09-22T23:26:50.947Z 19 380-399 51 51 51 76 76 76 F F F
68162023-09-22T23:26:50.950Z 20 400-419 51 51 51 76 76 76 F F F
68172023-09-22T23:26:50.953Z 21 420-439 49 49 49 74 74 74 F F F
68182023-09-22T23:26:50.956Z 22 440-459 50 50 50 75 75 75 F F F
68192023-09-22T23:26:50.959Z 23 460-479 47 47 47 71 71 71 F F F
68202023-09-22T23:26:50.962Z 24 480-499 46 46 46 70 70 70 F F F
68212023-09-22T23:26:50.967Z 25 500-519 49 49 49 74 74 74 F F F
68222023-09-22T23:26:50.969Z 26 520-539 47 47 47 71 71 71 F F F
68232023-09-22T23:26:50.973Z 27 540-559 50 50 50 75 75 75 F F F
68242023-09-22T23:26:50.976Z 28 560-579 51 51 51 76 76 76 F F F
68252023-09-22T23:26:50.983Z 29 580-599 51 51 51 76 76 76 F F F
68262023-09-22T23:26:50.986ZMax gen: 51, Max flush: 76
68272023-09-22T23:26:50.990Z
68282023-09-22T23:26:50.994ZBegin loop 26
68292023-09-22T23:26:50.997ZDownstairs to restart: 1
68302023-09-22T23:26:51.000Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.920351459Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68312023-09-22T23:26:51.004Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.920718124Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68322023-09-22T23:26:51.007Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.920792671Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68332023-09-22T23:26:51.010Z{"msg":"Crucible stats registered with UUID: 35494ecb-40dc-40cc-9322-0dbe4c063c34","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.92088508Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68342023-09-22T23:26:51.014Z{"msg":"Crucible runtime is spawned
68352023-09-22T23:26:51.017ZCrucible 35494ecb-40dc-40cc-9322-0dbe4c063c34 has session id: 1fff131a-5d60-4f30-b7ea-b2372fcada42","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.921006377Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68362023-09-22T23:26:51.020Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30The guest has requested activation with gen:52
68372023-09-22T23:26:51.023Z,"time":"2023-09-22T23:26:50.921078822Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370,"task":"up_listen"}
68382023-09-22T23:26:51.026Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.921120773Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68392023-09-22T23:26:51.030Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.921152515Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68402023-09-22T23:26:51.033Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.921269704Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370,"looper":"0"}
68412023-09-22T23:26:51.036Z{"msg":"35494ecb-40dc-40cc-9322-0dbe4c063c34 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.92134823Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68422023-09-22T23:26:51.039Z{"msg":"Set desired generation to :52","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.921409869Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68432023-09-22T23:26:51.043Z{"msg":"[0] 35494ecb-40dc-40cc-9322-0dbe4c063c34 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.921554341Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370,"looper":"0"}
68442023-09-22T23:26:51.046Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.921633266Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370,"looper":"1"}
68452023-09-22T23:26:51.049Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.921720829Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370,"looper":"2"}
68462023-09-22T23:26:51.052Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.921843815Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68472023-09-22T23:26:51.055Z{"msg":"[2] 35494ecb-40dc-40cc-9322-0dbe4c063c34 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.921937376Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370,"looper":"2"}
68482023-09-22T23:26:51.058Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.922069146Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68492023-09-22T23:26:51.061Z{"msg":"[0] 35494ecb-40dc-40cc-9322-0dbe4c063c34 (ba0af83b-a13e-4fc6-bfa8-f43e2c9c1e08) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.922712978Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68502023-09-22T23:26:51.064Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.922768899Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68512023-09-22T23:26:51.067Z{"msg":"[0] client is_active_req TRUE, promote! session ba0af83b-a13e-4fc6-bfa8-f43e2c9c1e08","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.92286126Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68522023-09-22T23:26:51.071Z{"msg":"[2] 35494ecb-40dc-40cc-9322-0dbe4c063c34 (ba0af83b-a13e-4fc6-bfa8-f43e2c9c1e08) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.922932327Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68532023-09-22T23:26:51.074Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.922965837Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68542023-09-22T23:26:51.077Z{"msg":"[2] client is_active_req TRUE, promote! session ba0af83b-a13e-4fc6-bfa8-f43e2c9c1e08","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.923031497Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68552023-09-22T23:26:51.080Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.923507044Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68562023-09-22T23:26:51.084Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.923569967Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68572023-09-22T23:26:51.087Z{"msg":"35494ecb-40dc-40cc-9322-0dbe4c063c34 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.923618872Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68582023-09-22T23:26:51.090Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.923729293Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68592023-09-22T23:26:51.093Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.923784008Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68602023-09-22T23:26:51.097Z{"msg":"35494ecb-40dc-40cc-9322-0dbe4c063c34 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.923831297Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68612023-09-22T23:26:51.100Z{"msg":"[0] 35494ecb-40dc-40cc-9322-0dbe4c063c34 (ba0af83b-a13e-4fc6-bfa8-f43e2c9c1e08) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.925707768Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68622023-09-22T23:26:51.103Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.925767974Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68632023-09-22T23:26:51.106Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:50.925790244Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68642023-09-22T23:26:51.109Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.925883007Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68652023-09-22T23:26:51.112Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.925930909Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68662023-09-22T23:26:51.115Z{"msg":"35494ecb-40dc-40cc-9322-0dbe4c063c34 WaitQuorum New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.925978899Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68672023-09-22T23:26:51.118Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.926001238Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68682023-09-22T23:26:51.121Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.926038648Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68692023-09-22T23:26:51.125Z{"msg":"[2] 35494ecb-40dc-40cc-9322-0dbe4c063c34 (ba0af83b-a13e-4fc6-bfa8-f43e2c9c1e08) WaitQuorum New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.926096992Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68702023-09-22T23:26:51.128Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.92614016Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68712023-09-22T23:26:51.131Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:50.92617974Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68722023-09-22T23:26:51.134Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.926200804Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68732023-09-22T23:26:51.137Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.926238457Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68742023-09-22T23:26:51.140Z{"msg":"35494ecb-40dc-40cc-9322-0dbe4c063c34 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.926277983Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68752023-09-22T23:26:51.143Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.926297865Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68762023-09-22T23:26:51.146Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:50.926315135Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68772023-09-22T23:26:51.925Z{"msg":"[1] 35494ecb-40dc-40cc-9322-0dbe4c063c34 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.923358989Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370,"looper":"1"}
68782023-09-22T23:26:51.929Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.923443961Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68792023-09-22T23:26:51.932Z{"msg":"[1] 35494ecb-40dc-40cc-9322-0dbe4c063c34 (ba0af83b-a13e-4fc6-bfa8-f43e2c9c1e08) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.924499989Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68802023-09-22T23:26:51.935Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.924553166Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68812023-09-22T23:26:51.938Z{"msg":"[1] client is_active_req TRUE, promote! session ba0af83b-a13e-4fc6-bfa8-f43e2c9c1e08","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.924596262Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68822023-09-22T23:26:51.941Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.925192263Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68832023-09-22T23:26:51.944Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.925257506Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68842023-09-22T23:26:51.947Z{"msg":"35494ecb-40dc-40cc-9322-0dbe4c063c34 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.925301986Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68852023-09-22T23:26:51.950Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.927634582Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68862023-09-22T23:26:51.953Z{"msg":"[1] 35494ecb-40dc-40cc-9322-0dbe4c063c34 (ba0af83b-a13e-4fc6-bfa8-f43e2c9c1e08) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.927874295Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68872023-09-22T23:26:51.956Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.927938774Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68882023-09-22T23:26:51.959Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:51.927982065Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68892023-09-22T23:26:51.962Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.928024017Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68902023-09-22T23:26:51.965Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.928057997Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68912023-09-22T23:26:51.968Z{"msg":"35494ecb-40dc-40cc-9322-0dbe4c063c34 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.928123803Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68922023-09-22T23:26:51.972Z{"msg":"[0]R flush_numbers[0..12]: [67, 76, 58, 74, 65, 74, 73, 74, 74, 75, 70, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.928189723Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68932023-09-22T23:26:51.975Z{"msg":"[0]R generation[0..12]: [43, 51, 39, 49, 42, 49, 48, 49, 49, 50, 46, 35]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:52
68942023-09-22T23:26:51.978Z,"time":"2023-09-22T23:26:51.928276265Z","Wait for a query_work_queue command to finish before sending IO
68952023-09-22T23:26:51.981Zhostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68962023-09-22T23:26:51.984Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.928316348Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68972023-09-22T23:26:51.987Z{"msg":"[1]R flush_numbers[0..12]: [67, 76, 58, 74, 65, 74, 73, 74, 74, 75, 70, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.92837927Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68982023-09-22T23:26:51.990Z{"msg":"[1]R generation[0..12]: [43, 51, 39, 49, 42, 49, 48, 49, 49, 50, 46, 35]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.928479099Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
68992023-09-22T23:26:51.993Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.928541461Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
69002023-09-22T23:26:51.996Z{"msg":"[2]R flush_numbers[0..12]: [67, 76, 58, 74, 65, 74, 73, 74, 74, 75, 70, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.9285983Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
69012023-09-22T23:26:52.000Z{"msg":"[2]R generation[0..12]: [43, 51, 39, 49, 42, 49, 48, 49, 49, 50, 46, 35]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.928669477Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
69022023-09-22T23:26:52.003Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.92874315Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
69032023-09-22T23:26:52.006Z{"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.928805661Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
69042023-09-22T23:26:52.009Z{"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.92886578Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
69052023-09-22T23:26:52.012Z{"msg":"Next flush: 77","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.928920543Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
69062023-09-22T23:26:52.015Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.928973624Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
69072023-09-22T23:26:52.019Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.929089095Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
69082023-09-22T23:26:52.022Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
69092023-09-22T23:26:52.025Z,"time":"2023-09-22T23:26:51.929147837Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
69102023-09-22T23:26:52.028Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.929286894Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
69112023-09-22T23:26:52.031Z{"msg":"35494ecb-40dc-40cc-9322-0dbe4c063c34 is now active with session: ba0af83b-a13e-4fc6-bfa8-f43e2c9c1e08","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.929349659Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
69122023-09-22T23:26:52.034Z{"msg":"35494ecb-40dc-40cc-9322-0dbe4c063c34 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.929405023Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
69132023-09-22T23:26:52.037Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.929459711Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
69142023-09-22T23:26:52.041Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.929512181Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
69152023-09-22T23:26:52.044Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.929564937Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
69162023-09-22T23:26:52.047Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.92961798Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
69172023-09-22T23:26:52.050Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.929671081Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
69182023-09-22T23:26:52.053Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.92972427Z","hostnameLoading write count information from file "/tmp/test_repair_verify.data"
69192023-09-22T23:26:52.055Z":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
69202023-09-22T23:26:52.059ZRun Repair workload
69212023-09-22T23:26:52.062Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.929843539Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
69222023-09-22T23:26:52.065Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.92990032Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
69232023-09-22T23:26:52.068Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.929953705Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
69242023-09-22T23:26:52.071Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.930007076Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
69252023-09-22T23:26:52.074Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:51.930080061Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1370}
69262023-09-22T23:26:52.078Z01/30 Write block 182 len 24576 data: 5 3 3 2 3 3
69272023-09-22T23:26:52.081Z02/30 Read block 7 len 16384
69282023-09-22T23:26:52.084Z03/30 Read block 2 len 8192
69292023-09-22T23:26:52.087Z04/30 Read block 69 len 32768
69302023-09-22T23:26:52.090Z05/30 Read block 88 len 8192
69312023-09-22T23:26:52.093Z06/30 Read block 173 len 20480
69322023-09-22T23:26:52.097Z07/30 Write block 491 len 28672 data: 7 6 7 6 6 6 6
69332023-09-22T23:26:52.100Z08/30 Write block 74 len 28672 data: 2 3 4 4 4 5 5
69342023-09-22T23:26:52.103Z09/30 Read block 96 len 24576
69352023-09-22T23:26:52.106Z10/30 Read block 588 len 36864
69362023-09-22T23:26:52.111Z11/30 Read block 445 len 12288
69372023-09-22T23:26:52.115Z12/30 Read block 343 len 16384
69382023-09-22T23:26:52.119Z13/30 Flush
69392023-09-22T23:26:52.122Z14/30 Write block 590 len 20480 data: 4 4 6 7 6
69402023-09-22T23:26:52.124Z15/30 Write block 83 len 40960 data: 6 5 5 5 6 5 5 5 5 5
69412023-09-22T23:26:52.128Z16/30 Read block 180 len 36864
69422023-09-22T23:26:52.130Z17/30 Write block 496 len 12288 data: 7 7 5
69432023-09-22T23:26:52.133Z18/30 Flush
69442023-09-22T23:26:52.136Z19/30 Write block 482 len 20480 data: 4 3 5 5 5
69452023-09-22T23:26:52.139Z20/30 Write block 301 len 28672 data: 6 6 7 8 7 7 7
69462023-09-22T23:26:52.142Z21/30 Flush
69472023-09-22T23:26:52.145Z22/30 Write block 320 len 8192 data: 8 8
69482023-09-22T23:26:52.148Z23/30 Read block 230 len 8192
69492023-09-22T23:26:52.151Z24/30 Write block 145 len 28672 data: 4 3 3 3 3 3 3
69502023-09-22T23:26:52.154Z25/30 Write block 568 len 32768 data: 8 8 5 4 3 3 3 3
69512023-09-22T23:26:52.159Z26/30 Write block 58 len 24576 data: 5 5 7 7 7 9
69522023-09-22T23:26:52.162Z27/30 Read block 374 len 16384
69532023-09-22T23:26:52.165Z28/30 Read block 233 len 16384
69542023-09-22T23:26:52.168Z29/30 Write block 450 len 16384 data: 8 8 8 7
69552023-09-22T23:26:52.170Z30/30 Read block 114 len 20480
69562023-09-22T23:26:52.173Z----------------------------------------------------------------
69572023-09-22T23:26:52.176Z Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30
69582023-09-22T23:26:52.179ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
69592023-09-22T23:26:52.182Z 1 Acked 1000 Write 6 Done Done Done false
69602023-09-22T23:26:52.184Z 2 Acked 1001 Read 4 Done Done Done false
69612023-09-22T23:26:52.187Z 3 Acked 1002 Read 2 Done Done Done false
69622023-09-22T23:26:52.191Z 4 Acked 1003 Read 8 Done Sent Done false
69632023-09-22T23:26:52.194Z 5 Acked 1004 Read 2 Done Sent Done false
69642023-09-22T23:26:52.198Z 6 Acked 1005 Read 5 Done Sent Done false
69652023-09-22T23:26:52.201Z 7 Acked 1006 Write 7 Done Sent Done false
69662023-09-22T23:26:52.203Z 8 Acked 1007 Write 7 Done Sent Done false
69672023-09-22T23:26:52.206Z 9 Acked 1008 Read 6 Done Sent Done false
69682023-09-22T23:26:52.209Z 10 Acked 1009 Read 9 Done Sent Done false
69692023-09-22T23:26:52.212Z 11 Acked 1010 Read 3 Done Sent Done false
69702023-09-22T23:26:52.215Z 12 Acked 1011 Read 4 Done Sent Done false
69712023-09-22T23:26:52.218Z 13 Acked 1012 Flush 0 Done Sent Done false
69722023-09-22T23:26:52.221Z 14 Acked 1013 Write 5 Done Sent Done false
69732023-09-22T23:26:52.224Z 15 Acked 1014 Write 10 Done Sent Done false
69742023-09-22T23:26:52.227Z 16 Acked 1015 Read 9 Done Sent Done false
69752023-09-22T23:26:52.230Z 17 Acked 1016 Write 3 Done Sent Done false
69762023-09-22T23:26:52.233Z 18 Acked 1017 Flush 0 Done Sent Done false
69772023-09-22T23:26:52.236Z 19 Acked 1018 Write 5 Done Sent Done false
69782023-09-22T23:26:52.239Z 20 Acked 1019 Write 7 Done Sent Done false
69792023-09-22T23:26:52.242Z 21 Acked 1020 Flush 0 Done Sent Done false
69802023-09-22T23:26:52.245Z 22 Acked 1021 Write 2 Done Sent Done false
69812023-09-22T23:26:52.249Z 23 Acked 1022 Read 2 Done Sent Done false
69822023-09-22T23:26:52.252Z 24 Acked 1023 Write 7 Done Sent Done false
69832023-09-22T23:26:52.255Z 25 Acked 1024 Write 8 Done Sent Done false
69842023-09-22T23:26:52.258Z 26 Acked 1025 Write 6 Done Sent Done false
69852023-09-22T23:26:52.261Z 27 Acked 1026 Read 4 Done Sent Done false
69862023-09-22T23:26:52.264Z 28 Acked 1027 Read 4 Done Sent Done false
69872023-09-22T23:26:52.271Z 29 Acked 1028 Write 4 Done Sent Sent false
69882023-09-22T23:26:52.274Z 30 Acked 1029 Read 5 Done Sent Sent false
69892023-09-22T23:26:52.277Z STATES DS:0 DS:1 DS:2 TOTAL
69902023-09-22T23:26:52.280Z New 0 0 0 0
69912023-09-22T23:26:52.283Z Sent 0 27 2 29
69922023-09-22T23:26:52.286Z Done 30 3 28 61
69932023-09-22T23:26:52.288Z Skipped 0 0 0 0
69942023-09-22T23:26:52.291Z Error 0 0 0 0
69952023-09-22T23:26:52.294ZLast Flush: 1020 0 1020
69962023-09-22T23:26:52.297ZDownstairs last five completed:
69972023-09-22T23:26:52.300ZUpstairs last five completed: 30 29 28 27 26 25
69982023-09-22T23:26:52.303ZWrote out file "/tmp/test_repair_verify.data"
69992023-09-22T23:26:52.306Z
70002023-09-22T23:26:52.309ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
70012023-09-22T23:26:52.312Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:52.066083058Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1371}
70022023-09-22T23:26:52.315Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:52.066373271Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1371}
70032023-09-22T23:26:52.319Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:52.066396581Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1371}
70042023-09-22T23:26:52.322Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:52.094678479Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1371}
70052023-09-22T23:26:52.325Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:52.09471113Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1371}
70062023-09-22T23:26:52.328Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:52.094730595Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1371}
70072023-09-22T23:26:52.331Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:52.113801869Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1371}
70082023-09-22T23:26:52.334Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:52.11383343Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1371}
70092023-09-22T23:26:52.337ZError: Difference in extent metadata found!
70102023-09-22T23:26:52.340Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:52.113852537Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1371}
70112023-09-22T23:26:52.343ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70122023-09-22T23:26:52.346Z 0 000-019 43 43 43 67 67 67 F F F
70132023-09-22T23:26:52.349Z 1 020-039 51 51 51 76 76 76 F F F
70142023-09-22T23:26:52.352Z 2 040-059 39 39 39 58 58 58 T F T <---
70152023-09-22T23:26:52.355Z 3 060-079 52 49 52 77 74 77 T F T <---
70162023-09-22T23:26:52.358Z 4 080-099 52 42 52 78 65 78 F F F <---
70172023-09-22T23:26:52.361Z 5 100-119 49 49 49 74 74 74 F F F
70182023-09-22T23:26:52.364Z 6 120-139 48 48 48 73 73 73 F F F
70192023-09-22T23:26:52.367Z 7 140-159 49 49 49 74 74 74 T F T <---
70202023-09-22T23:26:52.370Z 8 160-179 49 49 49 74 74 74 F F F
70212023-09-22T23:26:52.373Z 9 180-199 52 50 52 77 75 77 F T F <---
70222023-09-22T23:26:52.376Z 10 200-219 46 46 46 70 70 70 F F F
70232023-09-22T23:26:52.379Z 11 220-239 35 35 35 52 52 52 F F F
70242023-09-22T23:26:52.382Z 12 240-259 42 42 42 64 64 64 F F F
70252023-09-22T23:26:52.385Z 13 260-279 51 51 51 76 76 76 F F F
70262023-09-22T23:26:52.387Z 14 280-299 48 48 48 73 73 73 F F F
70272023-09-22T23:26:52.390Z 15 300-319 52 49 52 79 74 79 F F F <---
70282023-09-22T23:26:52.394Z 16 320-339 51 51 51 76 76 76 T F T <---
70292023-09-22T23:26:52.396Z 17 340-359 48 48 48 72 72 72 F F F
70302023-09-22T23:26:52.399Z 18 360-379 45 45 45 69 69 69 F F F
70312023-09-22T23:26:52.403Z 19 380-399 51 51 51 76 76 76 F F F
70322023-09-22T23:26:52.406Z 20 400-419 51 51 51 76 76 76 F F F
70332023-09-22T23:26:52.409Z 21 420-439 49 49 49 74 74 74 F F F
70342023-09-22T23:26:52.413Z 22 440-459 50 50 50 75 75 75 T F T <---
70352023-09-22T23:26:52.416Z 23 460-479 47 47 47 71 71 71 F F F
70362023-09-22T23:26:52.419Z 24 480-499 52 46 52 79 70 79 F F F <---
70372023-09-22T23:26:52.422Z 25 500-519 49 49 49 74 74 74 F F F
70382023-09-22T23:26:52.425Z 26 520-539 47 47 47 71 71 71 F F F
70392023-09-22T23:26:52.428Z 27 540-559 50 50 50 75 75 75 F F F
70402023-09-22T23:26:52.431Z 28 560-579 51 51 51 76 76 76 T F T <---
70412023-09-22T23:26:52.434Z 29 580-599 52 51 52 78 76 78 F F F <---
70422023-09-22T23:26:52.437ZMax gen: 52, Max flush: 79
70432023-09-22T23:26:52.440ZA Difference in extent metadata is expected here
70442023-09-22T23:26:52.443ZOn loop 26
70452023-09-22T23:26:52.448Z
70462023-09-22T23:26:52.451ZVerifying data now
70472023-09-22T23:26:58.379ZNo change after verify
70482023-09-22T23:26:58.382ZLoop: 26 Downstairs dump after verify (and repair):
70492023-09-22T23:26:58.397Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.396629214Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1376}
70502023-09-22T23:26:58.400Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.396956819Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1376}
70512023-09-22T23:26:58.403Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.396981446Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1376}
70522023-09-22T23:26:58.417Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.415951462Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1376}
70532023-09-22T23:26:58.420Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.416005361Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1376}
70542023-09-22T23:26:58.423Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.416026159Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1376}
70552023-09-22T23:26:58.431Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.427298033Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1376}
70562023-09-22T23:26:58.434Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.427350651Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1376}
70572023-09-22T23:26:58.437Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.427371251Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1376}
70582023-09-22T23:26:58.440ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
70592023-09-22T23:26:58.443Z 0 000-019 43 43 43 67 67 67 F F F
70602023-09-22T23:26:58.445Z 1 020-039 51 51 51 76 76 76 F F F
70612023-09-22T23:26:58.448Z 2 040-059 53 53 53 80 80 80 F F F
70622023-09-22T23:26:58.452Z 3 060-079 53 53 53 80 80 80 F F F
70632023-09-22T23:26:58.454Z 4 080-099 52 52 52 78 78 78 F F F
70642023-09-22T23:26:58.457Z 5 100-119 49 49 49 74 74 74 F F F
70652023-09-22T23:26:58.460Z 6 120-139 48 48 48 73 73 73 F F F
70662023-09-22T23:26:58.463Z 7 140-159 53 53 53 80 80 80 F F F
70672023-09-22T23:26:58.466Z 8 160-179 49 49 49 74 74 74 F F F
70682023-09-22T23:26:58.469Z 9 180-199 52 52 52 77 77 77 F F F
70692023-09-22T23:26:58.472Z 10 200-219 46 46 46 70 70 70 F F F
70702023-09-22T23:26:58.475Z 11 220-239 35 35 35 52 52 52 F F F
70712023-09-22T23:26:58.477Z 12 240-259 42 42 42 64 64 64 F F F
70722023-09-22T23:26:58.480Z 13 260-279 51 51 51 76 76 76 F F F
70732023-09-22T23:26:58.485Z 14 280-299 48 48 48 73 73 73 F F F
70742023-09-22T23:26:58.488Z 15 300-319 52 52 52 79 79 79 F F F
70752023-09-22T23:26:58.491Z 16 320-339 53 53 53 80 80 80 F F F
70762023-09-22T23:26:58.493Z 17 340-359 48 48 48 72 72 72 F F F
70772023-09-22T23:26:58.496Z 18 360-379 45 45 45 69 69 69 F F F
70782023-09-22T23:26:58.499Z 19 380-399 51 51 51 76 76 76 F F F
70792023-09-22T23:26:58.502Z 20 400-419 51 51 51 76 76 76 F F F
70802023-09-22T23:26:58.505Z 21 420-439 49 49 49 74 74 74 F F F
70812023-09-22T23:26:58.510Z 22 440-459 53 53 53 80 80 80 F F F
70822023-09-22T23:26:58.512Z 23 460-479 47 47 47 71 71 71 F F F
70832023-09-22T23:26:58.515Z 24 480-499 52 52 52 79 79 79 F F F
70842023-09-22T23:26:58.518Z 25 500-519 49 49 49 74 74 74 F F F
70852023-09-22T23:26:58.521Z 26 520-539 47 47 47 71 71 71 F F F
70862023-09-22T23:26:58.524Z 27 540-559 50 50 50 75 75 75 F F F
70872023-09-22T23:26:58.527Z 28 560-579 53 53 53 80 80 80 F F F
70882023-09-22T23:26:58.530Z 29 580-599 52 52 52 78 78 78 F F F
70892023-09-22T23:26:58.533ZMax gen: 53, Max flush: 80
70902023-09-22T23:26:58.536Z
70912023-09-22T23:26:58.539ZBegin loop 27
70922023-09-22T23:26:58.542ZDownstairs to restart: 0
70932023-09-22T23:26:58.545Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.477681289Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
70942023-09-22T23:26:58.548Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.478050633Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
70952023-09-22T23:26:58.551Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.478137362Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
70962023-09-22T23:26:58.554Z{"msg":"Crucible stats registered with UUID: 30b3afa0-8191-4312-8e3a-49103fe726e8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.478177576Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
70972023-09-22T23:26:58.557Z{"msg":"Crucible 30b3afa0-8191-4312-8e3a-49103fe726e8 has session id: c5eaee15-b067-4ae6-85a0-65a93b80e7f1","v":0,"name":"crucible","level":30Crucible runtime is spawned
70982023-09-22T23:26:58.560Z,"time":"2023-09-22T23:26:58.478288852Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
70992023-09-22T23:26:58.563Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.478396964Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378,"task":"up_listen"}
71002023-09-22T23:26:58.566ZThe guest has requested activation with gen:54
71012023-09-22T23:26:58.569Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.478464179Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71022023-09-22T23:26:58.572Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.478529087Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71032023-09-22T23:26:58.575Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.478594918Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378,"looper":"2"}
71042023-09-22T23:26:58.578Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.478916437Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378,"looper":"1"}
71052023-09-22T23:26:58.581Z{"msg":"[2] 30b3afa0-8191-4312-8e3a-49103fe726e8 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.47900374Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378,"looper":"2"}
71062023-09-22T23:26:58.584Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.479039355Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378,"looper":"0"}
71072023-09-22T23:26:58.587Z{"msg":"[1] 30b3afa0-8191-4312-8e3a-49103fe726e8 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.479128136Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378,"looper":"1"}
71082023-09-22T23:26:58.590Z{"msg":"30b3afa0-8191-4312-8e3a-49103fe726e8 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.47954517Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71092023-09-22T23:26:58.593Z{"msg":"Set desired generation to :54","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.47960387Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71102023-09-22T23:26:58.596Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.479646003Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71112023-09-22T23:26:58.599Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.480058991Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71122023-09-22T23:26:58.601Z{"msg":"[2] 30b3afa0-8191-4312-8e3a-49103fe726e8 (a4d8a27c-fe1a-427b-9290-0b49ec923fe6) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.480409086Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71132023-09-22T23:26:58.604Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.480494171Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71142023-09-22T23:26:58.607Z{"msg":"[2] client is_active_req TRUE, promote! session a4d8a27c-fe1a-427b-9290-0b49ec923fe6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.48057002Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71152023-09-22T23:26:58.610Z{"msg":"[1] 30b3afa0-8191-4312-8e3a-49103fe726e8 (a4d8a27c-fe1a-427b-9290-0b49ec923fe6) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.480653969Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71162023-09-22T23:26:58.613Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.480725946Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71172023-09-22T23:26:58.617Z{"msg":"[1] client is_active_req TRUE, promote! session a4d8a27c-fe1a-427b-9290-0b49ec923fe6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.480808123Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71182023-09-22T23:26:58.620Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.481051185Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71192023-09-22T23:26:58.623Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.481165985Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71202023-09-22T23:26:58.626Z{"msg":"30b3afa0-8191-4312-8e3a-49103fe726e8 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.481264767Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71212023-09-22T23:26:58.629Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.48131025Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71222023-09-22T23:26:58.632Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.481336405Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71232023-09-22T23:26:58.635Z{"msg":"30b3afa0-8191-4312-8e3a-49103fe726e8 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.48135589Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71242023-09-22T23:26:58.638Z{"msg":"[2] 30b3afa0-8191-4312-8e3a-49103fe726e8 (a4d8a27c-fe1a-427b-9290-0b49ec923fe6) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.483342235Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71252023-09-22T23:26:58.641Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.483409396Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71262023-09-22T23:26:58.644Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:58.483470287Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71272023-09-22T23:26:58.646Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.483524766Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71282023-09-22T23:26:58.649Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.483579208Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71292023-09-22T23:26:58.652Z{"msg":"30b3afa0-8191-4312-8e3a-49103fe726e8 New WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.48363386Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71302023-09-22T23:26:58.655Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.48368853Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71312023-09-22T23:26:58.658Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.483742098Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71322023-09-22T23:26:58.662Z{"msg":"[1] 30b3afa0-8191-4312-8e3a-49103fe726e8 (a4d8a27c-fe1a-427b-9290-0b49ec923fe6) New WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.483799915Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71332023-09-22T23:26:58.665Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.48385295Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71342023-09-22T23:26:58.668Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:58.483906059Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71352023-09-22T23:26:58.671Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.483959317Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71362023-09-22T23:26:58.674Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.484018369Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71372023-09-22T23:26:58.677Z{"msg":"30b3afa0-8191-4312-8e3a-49103fe726e8 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.484072706Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71382023-09-22T23:26:58.680Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.484126564Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71392023-09-22T23:26:58.683Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:58.48417979Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71402023-09-22T23:26:59.482Z{"msg":"[0] 30b3afa0-8191-4312-8e3a-49103fe726e8 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.481004956Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378,"looper":"0"}
71412023-09-22T23:26:59.485Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.481121682Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71422023-09-22T23:26:59.488Z{"msg":"[0] 30b3afa0-8191-4312-8e3a-49103fe726e8 (a4d8a27c-fe1a-427b-9290-0b49ec923fe6) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.482210412Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71432023-09-22T23:26:59.491Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.482313789Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71442023-09-22T23:26:59.494Z{"msg":"[0] client is_active_req TRUE, promote! session a4d8a27c-fe1a-427b-9290-0b49ec923fe6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.482395546Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71452023-09-22T23:26:59.497Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.482934622Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71462023-09-22T23:26:59.501Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.483014599Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71472023-09-22T23:26:59.504Z{"msg":"30b3afa0-8191-4312-8e3a-49103fe726e8 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.483084032Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71482023-09-22T23:26:59.507Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.484195574Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71492023-09-22T23:26:59.510Z{"msg":"[0] 30b3afa0-8191-4312-8e3a-49103fe726e8 (a4d8a27c-fe1a-427b-9290-0b49ec923fe6) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.48551427Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71502023-09-22T23:26:59.513Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.485575533Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71512023-09-22T23:26:59.516Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:26:59.485635283Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71522023-09-22T23:26:59.519Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.485690883Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71532023-09-22T23:26:59.522Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.48574702Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71542023-09-22T23:26:59.525Z{"msg":"30b3afa0-8191-4312-8e3a-49103fe726e8 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.485803488Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71552023-09-22T23:26:59.529Z{"msg":"[0]R flush_numbers[0..12]: [67, 76, 80, 80, 78, 74, 73, 80, 74, 77, 70, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.485860169Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71562023-09-22T23:26:59.532Z{"msg":"[0]R generation[0..12]: [43, 51, 53, 53, 52, 49, 48, 53, 49, 52, 46, 35]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:54
71572023-09-22T23:26:59.537ZWait for a query_work_queue command to finish before sending IO
71582023-09-22T23:26:59.541Z,"time":"2023-09-22T23:26:59.485947426Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71592023-09-22T23:26:59.545Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.48618985Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71602023-09-22T23:26:59.548Z{"msg":"[1]R flush_numbers[0..12]: [67, 76, 80, 80, 78, 74, 73, 80, 74, 77, 70, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.486250371Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71612023-09-22T23:26:59.552Z{"msg":"[1]R generation[0..12]: [43, 51, 53, 53, 52, 49, 48, 53, 49, 52, 46, 35]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.486315672Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71622023-09-22T23:26:59.555Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.486372284Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71632023-09-22T23:26:59.558Z{"msg":"[2]R flush_numbers[0..12]: [67, 76, 80, 80, 78, 74, 73, 80, 74, 77, 70, 52]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.486434829Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71642023-09-22T23:26:59.561Z{"msg":"[2]R generation[0..12]: [43, 51, 53, 53, 52, 49, 48, 53, 49, 52, 46, 35]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.486490213Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71652023-09-22T23:26:59.564Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.48655753Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71662023-09-22T23:26:59.567Z{"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.48666742Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71672023-09-22T23:26:59.570Z{"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.486724642Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71682023-09-22T23:26:59.573Z{"msg":"Next flush: 81","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.486794152Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71692023-09-22T23:26:59.576Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.486851551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71702023-09-22T23:26:59.579Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.486910153Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71712023-09-22T23:26:59.582Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.486968309Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71722023-09-22T23:26:59.585ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
71732023-09-22T23:26:59.588Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.487025768Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71742023-09-22T23:26:59.592Z{"msg":"30b3afa0-8191-4312-8e3a-49103fe726e8 is now active with session: a4d8a27c-fe1a-427b-9290-0b49ec923fe6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.48708252Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71752023-09-22T23:26:59.595Z{"msg":"30b3afa0-8191-4312-8e3a-49103fe726e8 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.487136863Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71762023-09-22T23:26:59.598Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.487192205Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71772023-09-22T23:26:59.601Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.48725034Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71782023-09-22T23:26:59.605Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.48727211Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71792023-09-22T23:26:59.609Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.487309309Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71802023-09-22T23:26:59.612Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.487361803Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71812023-09-22T23:26:59.615Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.487381365Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71822023-09-22T23:26:59.618Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.487418301Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71832023-09-22T23:26:59.621Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.487456189Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71842023-09-22T23:26:59.625Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.487494257Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71852023-09-22T23:26:59.628Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.487532068Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378}
71862023-09-22T23:26:59.631Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30Loading write count information from file "/tmp/test_repair_verify.data"
71872023-09-22T23:26:59.634Z,"time":"2023-09-22T23:26:59.487586884Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1378Run Repair workload
71882023-09-22T23:26:59.638Z}
71892023-09-22T23:26:59.641Z01/30 Write block 113 len 4096 data: 5
71902023-09-22T23:26:59.645Z02/30 Read block 117 len 40960
71912023-09-22T23:26:59.649Z03/30 Write block 551 len 24576 data: 8 7 7 4 4 5
71922023-09-22T23:26:59.652Z04/30 Write block 359 len 12288 data: 5 6 5
71932023-09-22T23:26:59.655Z05/30 Read block 429 len 32768
71942023-09-22T23:26:59.659Z06/30 Write block 205 len 16384 data: 5 6 6 8
71952023-09-22T23:26:59.662Z07/30 Read block 390 len 4096
71962023-09-22T23:26:59.666Z08/30 Read block 418 len 32768
71972023-09-22T23:26:59.669Z09/30 Write block 532 len 32768 data: 4 4 4 4 4 7 7 7
71982023-09-22T23:26:59.672Z10/30 Read block 456 len 20480
71992023-09-22T23:26:59.675Z11/30 Read block 306 len 28672
72002023-09-22T23:26:59.678Z12/30 Write block 478 len 36864 data: 6 6 5 4 5 4 6 6 6
72012023-09-22T23:26:59.681Z13/30 Flush
72022023-09-22T23:26:59.684Z14/30 Write block 34 len 20480 data: 5 6 9 9 9
72032023-09-22T23:26:59.687Z15/30 Write block 221 len 24576 data: 4 3 4 4 5 6
72042023-09-22T23:26:59.689Z16/30 Read block 235 len 16384
72052023-09-22T23:26:59.693Z17/30 Read block 341 len 4096
72062023-09-22T23:26:59.696Z18/30 Write block 75 len 28672 data: 4 5 5 5 6 6 8
72072023-09-22T23:26:59.699Z19/30 Write block 488 len 36864 data: 7 8 7 8 7 8 7 7 8
72082023-09-22T23:26:59.702Z20/30 Read block 537 len 28672
72092023-09-22T23:26:59.705Z21/30 Read block 287 len 32768
72102023-09-22T23:26:59.708Z22/30 Flush
72112023-09-22T23:26:59.711Z23/30 Write block 218 len 12288 data: 6 4 5
72122023-09-22T23:26:59.714Z24/30 Read block 416 len 20480
72132023-09-22T23:26:59.717Z25/30 Read block 135 len 24576
72142023-09-22T23:26:59.720Z26/30 Read block 44 len 36864
72152023-09-22T23:26:59.723Z27/30 Read block 539 len 28672
72162023-09-22T23:26:59.726Z28/30 Read block 458 len 12288
72172023-09-22T23:26:59.729Z29/30 Write block 77 len 4096 data: 6
72182023-09-22T23:26:59.732Z30/30 Write block 545 len 36864 data: 4 6 4 7 8 8 9 8 8
72192023-09-22T23:26:59.735Z----------------------------------------------------------------
72202023-09-22T23:26:59.739Z Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:30
72212023-09-22T23:26:59.741ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
72222023-09-22T23:26:59.745Z 1 Acked 1000 Write 1 Done Done Done false
72232023-09-22T23:26:59.747Z 2 Acked 1001 Read 10 Done Done Done false
72242023-09-22T23:26:59.750Z 3 Acked 1002 Write 6 Sent Done Done false
72252023-09-22T23:26:59.753Z 4 Acked 1003 Write 3 Sent Done Done false
72262023-09-22T23:26:59.756Z 5 Acked 1004 Read 8 Sent Done Done false
72272023-09-22T23:26:59.759Z 6 Acked 1005 Write 4 Sent Done Done false
72282023-09-22T23:26:59.762Z 7 Acked 1006 Read 1 Sent Done Done false
72292023-09-22T23:26:59.765Z 8 Acked 1007 Read 8 Sent Done Done false
72302023-09-22T23:26:59.768Z 9 Acked 1008 Write 8 Sent Done Done false
72312023-09-22T23:26:59.771Z 10 Acked 1009 Read 5 Sent Done Done false
72322023-09-22T23:26:59.774Z 11 Acked 1010 Read 7 Sent Done Done false
72332023-09-22T23:26:59.778Z 12 Acked 1011 Write 9 Sent Done Done false
72342023-09-22T23:26:59.781Z 13 Acked 1012 Flush 0 Sent Done Done false
72352023-09-22T23:26:59.783Z 14 Acked 1013 Write 5 Sent Done Done false
72362023-09-22T23:26:59.786Z 15 Acked 1014 Write 6 Sent Done Done false
72372023-09-22T23:26:59.789Z 16 Acked 1015 Read 4 Sent Done Done false
72382023-09-22T23:26:59.792Z 17 Acked 1016 Read 1 Sent Done Done false
72392023-09-22T23:26:59.817Z 18 Acked 1017 Write 7 Sent Done Done false
72402023-09-22T23:26:59.820Z 19 Acked 1018 Write 9 Sent Done Done false
72412023-09-22T23:26:59.826Z 20 Acked 1019 Read 7 Sent Done Done false
72422023-09-22T23:26:59.829Z 21 Acked 1020 Read 8 Sent Done Done false
72432023-09-22T23:26:59.833Z 22 Acked 1021 Flush 0 Sent Done Done false
72442023-09-22T23:26:59.835Z 23 Acked 1022 Write 3 Sent Done Done false
72452023-09-22T23:26:59.839Z 24 Acked 1023 Read 5 Sent Done Done false
72462023-09-22T23:26:59.842Z 25 Acked 1024 Read 6 Sent Done Done false
72472023-09-22T23:26:59.845Z 26 Acked 1025 Read 9 Sent Done Done false
72482023-09-22T23:26:59.848Z 27 Acked 1026 Read 7 Sent Done Done false
72492023-09-22T23:26:59.851Z 28 Acked 1027 Read 3 Sent Done Done false
72502023-09-22T23:26:59.854Z 29 Acked 1028 Write 1 Sent Done Done false
72512023-09-22T23:26:59.857Z 30 Acked 1029 Write 9 Sent Sent Sent false
72522023-09-22T23:26:59.860Z STATES DS:0 DS:1 DS:2 TOTAL
72532023-09-22T23:26:59.863Z New 0 0 0 0
72542023-09-22T23:26:59.866Z Sent 28 1 1 30
72552023-09-22T23:26:59.869Z Done 2 29 29 60
72562023-09-22T23:26:59.872Z Skipped 0 0 0 0
72572023-09-22T23:26:59.875Z Error 0 0 0 0
72582023-09-22T23:26:59.878ZLast Flush: 0 1021 1021
72592023-09-22T23:26:59.881ZDownstairs last five completed:
72602023-09-22T23:26:59.884ZUpstairs last five completed: 30 29 28 27 26 25
72612023-09-22T23:26:59.887ZWrote out file "/tmp/test_repair_verify.data"
72622023-09-22T23:26:59.889Z
72632023-09-22T23:26:59.893ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
72642023-09-22T23:26:59.895Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.627266028Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1379}
72652023-09-22T23:26:59.898Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.627639562Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1379}
72662023-09-22T23:26:59.901Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.627663258Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1379}
72672023-09-22T23:26:59.904Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.655845836Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1379}
72682023-09-22T23:26:59.907Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.655879022Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1379}
72692023-09-22T23:26:59.911Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.655898191Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1379}
72702023-09-22T23:26:59.914ZError: Difference in extent metadata found!
72712023-09-22T23:26:59.916Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.673934384Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1379}
72722023-09-22T23:26:59.919Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.673965003Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1379}
72732023-09-22T23:26:59.922Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:26:59.67398394Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1379}
72742023-09-22T23:26:59.926ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
72752023-09-22T23:26:59.929Z 0 000-019 43 43 43 67 67 67 F F F
72762023-09-22T23:26:59.932Z 1 020-039 51 54 54 76 82 82 F F F <---
72772023-09-22T23:26:59.934Z 2 040-059 53 53 53 80 80 80 F F F
72782023-09-22T23:26:59.938Z 3 060-079 53 54 54 80 82 82 F T T <---
72792023-09-22T23:26:59.940Z 4 080-099 52 54 54 78 82 82 F F F <---
72802023-09-22T23:26:59.943Z 5 100-119 49 54 54 74 81 81 T F F <---
72812023-09-22T23:26:59.946Z 6 120-139 48 48 48 73 73 73 F F F
72822023-09-22T23:26:59.949Z 7 140-159 53 53 53 80 80 80 F F F
72832023-09-22T23:26:59.952Z 8 160-179 49 49 49 74 74 74 F F F
72842023-09-22T23:26:59.955Z 9 180-199 52 52 52 77 77 77 F F F
72852023-09-22T23:26:59.958Z 10 200-219 46 54 54 70 81 81 F T T <---
72862023-09-22T23:26:59.961Z 11 220-239 35 54 54 52 82 82 F T T <---
72872023-09-22T23:26:59.965Z 12 240-259 42 42 42 64 64 64 F F F
72882023-09-22T23:26:59.968Z 13 260-279 51 51 51 76 76 76 F F F
72892023-09-22T23:26:59.972Z 14 280-299 48 48 48 73 73 73 F F F
72902023-09-22T23:26:59.975Z 15 300-319 52 52 52 79 79 79 F F F
72912023-09-22T23:26:59.978Z 16 320-339 53 53 53 80 80 80 F F F
72922023-09-22T23:26:59.980Z 17 340-359 48 54 54 72 81 81 F F F <---
72932023-09-22T23:26:59.983Z 18 360-379 45 54 54 69 81 81 F F F <---
72942023-09-22T23:26:59.986Z 19 380-399 51 51 51 76 76 76 F F F
72952023-09-22T23:26:59.989Z 20 400-419 51 51 51 76 76 76 F F F
72962023-09-22T23:26:59.992Z 21 420-439 49 49 49 74 74 74 F F F
72972023-09-22T23:26:59.995Z 22 440-459 53 53 53 80 80 80 F F F
72982023-09-22T23:26:59.998Z 23 460-479 47 54 54 71 81 81 F F F <---
72992023-09-22T23:27:00.001Z 24 480-499 52 54 54 79 82 82 F F F <---
73002023-09-22T23:27:00.004Z 25 500-519 49 49 49 74 74 74 F F F
73012023-09-22T23:27:00.007Z 26 520-539 47 54 54 71 81 81 F F F <---
73022023-09-22T23:27:00.011Z 27 540-559 50 54 54 75 81 81 F T T <---
73032023-09-22T23:27:00.014Z 28 560-579 53 53 53 80 80 80 F F F
73042023-09-22T23:27:00.017Z 29 580-599 52 52 52 78 78 78 F F F
73052023-09-22T23:27:00.020ZMax gen: 54, Max flush: 82
73062023-09-22T23:27:00.023ZA Difference in extent metadata is expected here
73072023-09-22T23:27:00.026ZOn loop 27
73082023-09-22T23:27:00.029Z
73092023-09-22T23:27:00.041ZVerifying data now
73102023-09-22T23:27:06.022ZNo change after verify
73112023-09-22T23:27:06.025ZLoop: 27 Downstairs dump after verify (and repair):
73122023-09-22T23:27:06.446Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.039831685Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1384}
73132023-09-22T23:27:06.449Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.04017217Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1384}
73142023-09-22T23:27:06.452Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.040197906Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1384}
73152023-09-22T23:27:06.455Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.058933837Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1384}
73162023-09-22T23:27:06.458Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.058982091Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1384}
73172023-09-22T23:27:06.463Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.059020341Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1384}
73182023-09-22T23:27:06.466Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.069886956Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1384}
73192023-09-22T23:27:06.469Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.069932789Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1384}
73202023-09-22T23:27:06.472Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.069970753Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1384}
73212023-09-22T23:27:06.476ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
73222023-09-22T23:27:06.479Z 0 000-019 43 43 43 67 67 67 F F F
73232023-09-22T23:27:06.482Z 1 020-039 54 54 54 82 82 82 F F F
73242023-09-22T23:27:06.485Z 2 040-059 53 53 53 80 80 80 F F F
73252023-09-22T23:27:06.488Z 3 060-079 55 55 55 83 83 83 F F F
73262023-09-22T23:27:06.491Z 4 080-099 54 54 54 82 82 82 F F F
73272023-09-22T23:27:06.495Z 5 100-119 54 54 54 81 81 81 F F F
73282023-09-22T23:27:06.497Z 6 120-139 48 48 48 73 73 73 F F F
73292023-09-22T23:27:06.500Z 7 140-159 53 53 53 80 80 80 F F F
73302023-09-22T23:27:06.504Z 8 160-179 49 49 49 74 74 74 F F F
73312023-09-22T23:27:06.507Z 9 180-199 52 52 52 77 77 77 F F F
73322023-09-22T23:27:06.510Z 10 200-219 55 55 55 83 83 83 F F F
73332023-09-22T23:27:06.513Z 11 220-239 55 55 55 83 83 83 F F F
73342023-09-22T23:27:06.516Z 12 240-259 42 42 42 64 64 64 F F F
73352023-09-22T23:27:06.519Z 13 260-279 51 51 51 76 76 76 F F F
73362023-09-22T23:27:06.522Z 14 280-299 48 48 48 73 73 73 F F F
73372023-09-22T23:27:06.525Z 15 300-319 52 52 52 79 79 79 F F F
73382023-09-22T23:27:06.528Z 16 320-339 53 53 53 80 80 80 F F F
73392023-09-22T23:27:06.532Z 17 340-359 54 54 54 81 81 81 F F F
73402023-09-22T23:27:06.946Z 18 360-379 54 54 54 81 81 81 F F F
73412023-09-22T23:27:06.968Z 19 380-399 51 51 51 76 76 76 F F F
73422023-09-22T23:27:06.972Z 20 400-419 51 51 51 76 76 76 F F F
73432023-09-22T23:27:06.975Z 21 420-439 49 49 49 74 74 74 F F F
73442023-09-22T23:27:06.978Z 22 440-459 53 53 53 80 80 80 F F F
73452023-09-22T23:27:06.981Z 23 460-479 54 54 54 81 81 81 F F F
73462023-09-22T23:27:06.984Z 24 480-499 54 54 54 82 82 82 F F F
73472023-09-22T23:27:06.987Z 25 500-519 49 49 49 74 74 74 F F F
73482023-09-22T23:27:06.992Z 26 520-539 54 54 54 81 81 81 F F F
73492023-09-22T23:27:06.995Z 27 540-559 55 55 55 83 83 83 F F F
73502023-09-22T23:27:06.998Z 28 560-579 53 53 53 80 80 80 F F F
73512023-09-22T23:27:07.001Z 29 580-599 52 52 52 78 78 78 F F F
73522023-09-22T23:27:07.004ZMax gen: 55, Max flush: 83
73532023-09-22T23:27:07.009Z
73542023-09-22T23:27:07.012ZBegin loop 28
73552023-09-22T23:27:07.016ZDownstairs to restart: 1
73562023-09-22T23:27:07.018Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.119197022Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73572023-09-22T23:27:07.021Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.119614506Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73582023-09-22T23:27:07.026Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.11967094Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73592023-09-22T23:27:07.029Z{"msg":"Crucible stats registered with UUID: 950c2c81-3ec8-4a0c-876d-5e02ac0b3abc","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.119738718Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73602023-09-22T23:27:07.031Z{"msg":"Crucible 950c2c81-3ec8-4a0c-876d-5e02ac0b3abc has session id: c6a194a9-15ea-475d-bed3-4e3658c8ead2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.119820003Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73612023-09-22T23:27:07.034ZCrucible runtime is spawned
73622023-09-22T23:27:07.037Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.120028848Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386,"task":"up_listen"}
73632023-09-22T23:27:07.040Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.120127983Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73642023-09-22T23:27:07.043Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30The guest has requested activation with gen:56
73652023-09-22T23:27:07.046Z,"time":"2023-09-22T23:27:06.120198539Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73662023-09-22T23:27:07.049Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.120309964Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386,"looper":"1"}
73672023-09-22T23:27:07.051Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.1204535Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386,"looper":"0"}
73682023-09-22T23:27:07.054Z{"msg":"[0] 950c2c81-3ec8-4a0c-876d-5e02ac0b3abc looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.120641716Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386,"looper":"0"}
73692023-09-22T23:27:07.057Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.120771612Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386,"looper":"2"}
73702023-09-22T23:27:07.060Z{"msg":"[2] 950c2c81-3ec8-4a0c-876d-5e02ac0b3abc looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.120883093Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386,"looper":"2"}
73712023-09-22T23:27:07.063Z{"msg":"950c2c81-3ec8-4a0c-876d-5e02ac0b3abc active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.121186078Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73722023-09-22T23:27:07.066Z{"msg":"Set desired generation to :56","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.121254587Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73732023-09-22T23:27:07.069Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.121356059Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73742023-09-22T23:27:07.072Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.121692219Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73752023-09-22T23:27:07.075Z{"msg":"[0] 950c2c81-3ec8-4a0c-876d-5e02ac0b3abc (3b6bac51-bae0-4428-857d-d9f1ed4a773d) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.122086355Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73762023-09-22T23:27:07.078Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.122157246Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73772023-09-22T23:27:07.081Z{"msg":"[0] client is_active_req TRUE, promote! session 3b6bac51-bae0-4428-857d-d9f1ed4a773d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.122250472Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73782023-09-22T23:27:07.084Z{"msg":"[2] 950c2c81-3ec8-4a0c-876d-5e02ac0b3abc (3b6bac51-bae0-4428-857d-d9f1ed4a773d) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.12232515Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73792023-09-22T23:27:07.087Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.122365132Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73802023-09-22T23:27:07.090Z{"msg":"[2] client is_active_req TRUE, promote! session 3b6bac51-bae0-4428-857d-d9f1ed4a773d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.122407124Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73812023-09-22T23:27:07.093Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.122724663Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73822023-09-22T23:27:07.096Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.122765555Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73832023-09-22T23:27:07.099Z{"msg":"950c2c81-3ec8-4a0c-876d-5e02ac0b3abc WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.12285591Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73842023-09-22T23:27:07.101Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.122952496Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73852023-09-22T23:27:07.104Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.122988443Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73862023-09-22T23:27:07.107Z{"msg":"950c2c81-3ec8-4a0c-876d-5e02ac0b3abc WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.123023022Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73872023-09-22T23:27:07.111Z{"msg":"[0] 950c2c81-3ec8-4a0c-876d-5e02ac0b3abc (3b6bac51-bae0-4428-857d-d9f1ed4a773d) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.124932676Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73882023-09-22T23:27:07.114Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.124986311Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73892023-09-22T23:27:07.117Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:27:06.125026947Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73902023-09-22T23:27:07.121Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.125065276Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73912023-09-22T23:27:07.124Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.125101842Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73922023-09-22T23:27:07.127Z{"msg":"950c2c81-3ec8-4a0c-876d-5e02ac0b3abc WaitQuorum New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.125135925Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73932023-09-22T23:27:07.130Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.125170344Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73942023-09-22T23:27:07.133Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.125204087Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73952023-09-22T23:27:07.136Z{"msg":"[2] 950c2c81-3ec8-4a0c-876d-5e02ac0b3abc (3b6bac51-bae0-4428-857d-d9f1ed4a773d) WaitQuorum New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.125247375Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73962023-09-22T23:27:07.139Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.125280954Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73972023-09-22T23:27:07.142Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:27:06.125310565Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73982023-09-22T23:27:07.145Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.125343554Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
73992023-09-22T23:27:07.149Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.125378493Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74002023-09-22T23:27:07.152Z{"msg":"950c2c81-3ec8-4a0c-876d-5e02ac0b3abc WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.1254122Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74012023-09-22T23:27:07.155Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.125441896Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74022023-09-22T23:27:07.158Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:06.125471227Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74032023-09-22T23:27:07.161Z{"msg":"[1] 950c2c81-3ec8-4a0c-876d-5e02ac0b3abc looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.122445769Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386,"looper":"1"}
74042023-09-22T23:27:07.164Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.122560212Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74052023-09-22T23:27:07.167Z{"msg":"[1] 950c2c81-3ec8-4a0c-876d-5e02ac0b3abc (3b6bac51-bae0-4428-857d-d9f1ed4a773d) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.123665424Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74062023-09-22T23:27:07.170Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.123758127Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74072023-09-22T23:27:07.173Z{"msg":"[1] client is_active_req TRUE, promote! session 3b6bac51-bae0-4428-857d-d9f1ed4a773d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.123817718Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74082023-09-22T23:27:07.176Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.124356243Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74092023-09-22T23:27:07.179Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.124413795Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74102023-09-22T23:27:07.181Z{"msg":"950c2c81-3ec8-4a0c-876d-5e02ac0b3abc WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.124456952Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74112023-09-22T23:27:07.184Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.126605933Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74122023-09-22T23:27:07.596Z{"msg":"[1] 950c2c81-3ec8-4a0c-876d-5e02ac0b3abc (3b6bac51-bae0-4428-857d-d9f1ed4a773d) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.126831733Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74132023-09-22T23:27:07.599Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.126886282Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74142023-09-22T23:27:07.602Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:27:07.126934923Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74152023-09-22T23:27:07.605Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.126974094Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74162023-09-22T23:27:07.609Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.127013318Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74172023-09-22T23:27:07.612Z{"msg":"950c2c81-3ec8-4a0c-876d-5e02ac0b3abc WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.12705286Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74182023-09-22T23:27:07.615Z{"msg":"[0]R flush_numbers[0..12]: [67, 82, 80, 83, 82, 81, 73, 80, 74, 77, 83, 83]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.127093283Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74192023-09-22T23:27:07.619Z{"msg":"[0]R generation[0..12]: [43, 54, 53, 55, 54, 54, 48, 53, 49, 52, 55, 55]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.127133305Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74202023-09-22T23:27:07.622Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:56
74212023-09-22T23:27:07.625ZWait for a query_work_queue command to finish before sending IO
74222023-09-22T23:27:07.627Z,"time":"2023-09-22T23:27:07.127252757Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74232023-09-22T23:27:07.630Z{"msg":"[1]R flush_numbers[0..12]: [67, 82, 80, 83, 82, 81, 73, 80, 74, 77, 83, 83]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.127481239Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74242023-09-22T23:27:07.633Z{"msg":"[1]R generation[0..12]: [43, 54, 53, 55, 54, 54, 48, 53, 49, 52, 55, 55]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.12755996Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74252023-09-22T23:27:07.636Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.127610411Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74262023-09-22T23:27:07.639Z{"msg":"[2]R flush_numbers[0..12]: [67, 82, 80, 83, 82, 81, 73, 80, 74, 77, 83, 83]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.127652067Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74272023-09-22T23:27:07.643Z{"msg":"[2]R generation[0..12]: [43, 54, 53, 55, 54, 54, 48, 53, 49, 52, 55, 55]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.127700015Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74282023-09-22T23:27:07.646Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.127740474Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74292023-09-22T23:27:07.649Z{"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.127785344Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74302023-09-22T23:27:07.652Z{"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.127834217Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74312023-09-22T23:27:07.655Z{"msg":"Next flush: 84","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.127872506Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74322023-09-22T23:27:07.659Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.127911074Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74332023-09-22T23:27:07.662Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.127953578Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74342023-09-22T23:27:07.664Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.127993271Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74352023-09-22T23:27:07.668Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.128032118Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74362023-09-22T23:27:07.672Z{"msg":"950c2c81-3ec8-4a0c-876d-5e02ac0b3abc is now active with session: 3b6bac51-bae0-4428-857d-d9f1ed4a773d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.12807765Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74372023-09-22T23:27:07.675Z{"msg":"950c2c81-3ec8-4a0c-876d-5e02ac0b3abc Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.128125927Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74382023-09-22T23:27:07.678Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.128172753Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74392023-09-22T23:27:07.681ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
74402023-09-22T23:27:07.684Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.128216421Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74412023-09-22T23:27:07.687Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.128255677Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74422023-09-22T23:27:07.690Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.128293529Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74432023-09-22T23:27:07.693Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.128331278Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74442023-09-22T23:27:07.696Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.128369369Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74452023-09-22T23:27:07.700Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.128407332Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74462023-09-22T23:27:07.703Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.128445786Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74472023-09-22T23:27:07.707Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.128483751Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74482023-09-22T23:27:07.710Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.128521777Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74492023-09-22T23:27:07.713Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.12855935Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1386}
74502023-09-22T23:27:07.716ZLoading write count information from file "/tmp/test_repair_verify.data"
74512023-09-22T23:27:07.719ZRun Repair workload
74522023-09-22T23:27:07.722Z01/30 Write block 589 len 32768 data: 4 5 5 7 8 7 7 7
74532023-09-22T23:27:07.725Z02/30 Flush
74542023-09-22T23:27:07.728Z03/30 Flush
74552023-09-22T23:27:07.732Z04/30 Write block 276 len 40960 data: 11 7 10 10 9 8 8 8 6 6
74562023-09-22T23:27:07.734Z05/30 Read block 507 len 36864
74572023-09-22T23:27:07.737Z06/30 Read block 444 len 12288
74582023-09-22T23:27:07.740Z07/30 Read block 44 len 16384
74592023-09-22T23:27:07.743Z08/30 Read block 57 len 40960
74602023-09-22T23:27:07.746Z09/30 Write block 333 len 16384 data: 3 5 6 6
74612023-09-22T23:27:07.749Z10/30 Write block 178 len 16384 data: 5 5 3 4
74622023-09-22T23:27:07.752Z11/30 Write block 99 len 40960 data: 5 4 4 3 4 5 7 7 7 6
74632023-09-22T23:27:07.755Z12/30 Write block 506 len 12288 data: 6 6 5
74642023-09-22T23:27:07.758Z13/30 Read block 396 len 12288
74652023-09-22T23:27:07.761Z14/30 Write block 65 len 4096 data: 10
74662023-09-22T23:27:07.764Z15/30 Write block 200 len 36864 data: 4 3 5 5 4 6 7 7 9
74672023-09-22T23:27:07.767Z16/30 Read block 232 len 8192
74682023-09-22T23:27:07.770Z17/30 Write block 45 len 36864 data: 7 6 7 7 6 5 5 3 2
74692023-09-22T23:27:07.773Z18/30 Read block 483 len 24576
74702023-09-22T23:27:07.776Z19/30 Write block 280 len 36864 data: 10 9 9 9 7 7 7 7 7
74712023-09-22T23:27:07.779Z20/30 Read block 362 len 28672
74722023-09-22T23:27:07.782Z21/30 Write block 523 len 32768 data: 4 3 5 6 6 5 5 3
74732023-09-22T23:27:07.785Z22/30 Read block 469 len 36864
74742023-09-22T23:27:07.790Z23/30 Write block 138 len 24576 data: 7 7 7 7 6 6
74752023-09-22T23:27:07.793Z24/30 Read block 320 len 32768
74762023-09-22T23:27:07.795Z25/30 Read block 225 len 32768
74772023-09-22T23:27:07.798Z26/30 Read block 21 len 36864
74782023-09-22T23:27:07.802ZError: Difference in extent metadata found!
74792023-09-22T23:27:07.805Z27/30 Read block 246 len 4096
74802023-09-22T23:27:07.808Z28/30 Write block 63 len 24576 data: 10 9 11 10 10 11
74812023-09-22T23:27:07.810Z29/30 Write block 499 len 36864 data: 5 6 5 6 5 5 5 7 7
74822023-09-22T23:27:07.813Z30/30 Read block 286 len 28672
74832023-09-22T23:27:07.816Z----------------------------------------------------------------
74842023-09-22T23:27:07.819Z Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:28
74852023-09-22T23:27:07.822ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
74862023-09-22T23:27:07.825Z 3 Acked 1002 Flush 0 Done Sent Done false
74872023-09-22T23:27:07.828Z 4 Acked 1003 Write 10 Done Sent Done false
74882023-09-22T23:27:07.831Z 5 Acked 1004 Read 9 Done Sent Done false
74892023-09-22T23:27:07.834Z 6 Acked 1005 Read 3 Done Sent Done false
74902023-09-22T23:27:07.837Z 7 Acked 1006 Read 4 Done Sent Done false
74912023-09-22T23:27:07.840Z 8 Acked 1007 Read 10 Done Sent Done false
74922023-09-22T23:27:07.843Z 9 Acked 1008 Write 4 Done Sent Done false
74932023-09-22T23:27:07.845Z 10 Acked 1009 Write 4 Done Sent Done false
74942023-09-22T23:27:07.848Z 11 Acked 1010 Write 10 Done Sent Done false
74952023-09-22T23:27:07.851Z 12 Acked 1011 Write 3 Done Sent Done false
74962023-09-22T23:27:07.854Z 13 Acked 1012 Read 3 Done Sent Done false
74972023-09-22T23:27:07.857Z 14 Acked 1013 Write 1 Done Sent Done false
74982023-09-22T23:27:07.860Z 15 Acked 1014 Write 9 Done Sent Done false
74992023-09-22T23:27:07.863Z 16 Acked 1015 Read 2 Done Sent Done false
75002023-09-22T23:27:07.866Z 17 Acked 1016 Write 9 Done Sent Done false
75012023-09-22T23:27:07.868Z 18 Acked 1017 Read 6 Done Sent Done false
75022023-09-22T23:27:07.871Z 19 Acked 1018 Write 9 Done Sent Done false
75032023-09-22T23:27:07.874Z 20 Acked 1019 Read 7 Done Sent Done false
75042023-09-22T23:27:07.877Z 21 Acked 1020 Write 8 Done Sent Done false
75052023-09-22T23:27:07.880Z 22 Acked 1021 Read 9 Done Sent Done false
75062023-09-22T23:27:07.883Z 23 Acked 1022 Write 6 Done Sent Done false
75072023-09-22T23:27:07.886Z 24 Acked 1023 Read 8 Done Sent Done false
75082023-09-22T23:27:07.889Z 25 Acked 1024 Read 8 Done Sent Done false
75092023-09-22T23:27:07.891Z 26 Acked 1025 Read 9 Done Sent Done false
75102023-09-22T23:27:07.894Z 27 Acked 1026 Read 1 Done Sent Done false
75112023-09-22T23:27:07.897Z 28 Acked 1027 Write 6 Done Sent Done false
75122023-09-22T23:27:07.900Z 29 Acked 1028 Write 9 Done Sent Sent false
75132023-09-22T23:27:07.904Z 30 Acked 1029 Read 7 Done Sent Sent false
75142023-09-22T23:27:07.906Z STATES DS:0 DS:1 DS:2 TOTAL
75152023-09-22T23:27:07.909Z New 0 0 0 0
75162023-09-22T23:27:08.316Z Sent 0 28 2 30
75172023-09-22T23:27:08.319Z Done 28 0 26 54
75182023-09-22T23:27:08.322Z Skipped 0 0 0 0
75192023-09-22T23:27:08.325Z Error 0 0 0 0
75202023-09-22T23:27:08.328ZLast Flush: 1002 1001 1002
75212023-09-22T23:27:08.331ZDownstairs last five completed: 1001 1000
75222023-09-22T23:27:08.334ZUpstairs last five completed: 30 29 28 27 26 25
75232023-09-22T23:27:08.337ZWrote out file "/tmp/test_repair_verify.data"
75242023-09-22T23:27:08.339Z
75252023-09-22T23:27:08.345ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
75262023-09-22T23:27:08.348Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.278200717Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1387}
75272023-09-22T23:27:08.351Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.278453878Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1387}
75282023-09-22T23:27:08.354Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.2784767Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1387}
75292023-09-22T23:27:08.357Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.318204192Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1387}
75302023-09-22T23:27:08.360Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.318233928Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1387}
75312023-09-22T23:27:08.363Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.318252586Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1387}
75322023-09-22T23:27:08.366Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.335239183Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1387}
75332023-09-22T23:27:08.368Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.335267194Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1387}
75342023-09-22T23:27:08.372Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:07.335285948Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1387}
75352023-09-22T23:27:08.375ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
75362023-09-22T23:27:08.380Z 0 000-019 43 43 43 67 67 67 F F F
75372023-09-22T23:27:08.384Z 1 020-039 54 54 54 82 82 82 F F F
75382023-09-22T23:27:08.387Z 2 040-059 53 53 53 80 80 80 T F T <---
75392023-09-22T23:27:08.390Z 3 060-079 55 55 55 83 83 83 T F T <---
75402023-09-22T23:27:08.395Z 4 080-099 54 54 54 82 82 82 T F T <---
75412023-09-22T23:27:08.397Z 5 100-119 54 54 54 81 81 81 T F T <---
75422023-09-22T23:27:08.401Z 6 120-139 48 48 48 73 73 73 T F T <---
75432023-09-22T23:27:08.406Z 7 140-159 53 53 53 80 80 80 T F T <---
75442023-09-22T23:27:08.410Z 8 160-179 49 49 49 74 74 74 T F T <---
75452023-09-22T23:27:08.413Z 9 180-199 52 52 52 77 77 77 T F T <---
75462023-09-22T23:27:08.417Z 10 200-219 55 55 55 83 83 83 T F T <---
75472023-09-22T23:27:08.422Z 11 220-239 55 55 55 83 83 83 F F F
75482023-09-22T23:27:08.425Z 12 240-259 42 42 42 64 64 64 F F F
75492023-09-22T23:27:08.428Z 13 260-279 51 51 51 76 76 76 T F T <---
75502023-09-22T23:27:08.432Z 14 280-299 48 48 48 73 73 73 T F T <---
75512023-09-22T23:27:08.434Z 15 300-319 52 52 52 79 79 79 F F F
75522023-09-22T23:27:08.437Z 16 320-339 53 53 53 80 80 80 T F T <---
75532023-09-22T23:27:08.440Z 17 340-359 54 54 54 81 81 81 F F F
75542023-09-22T23:27:08.443Z 18 360-379 54 54 54 81 81 81 F F F
75552023-09-22T23:27:08.446Z 19 380-399 51 51 51 76 76 76 F F F
75562023-09-22T23:27:08.856Z 20 400-419 51 51 51 76 76 76 F F F
75572023-09-22T23:27:08.859Z 21 420-439 49 49 49 74 74 74 F F F
75582023-09-22T23:27:09.265Z 22 440-459 53 53 53 80 80 80 F F F
75592023-09-22T23:27:09.269Z 23 460-479 54 54 54 81 81 81 F F F
75602023-09-22T23:27:09.272Z 24 480-499 54 54 54 82 82 82 T F T <---
75612023-09-22T23:27:09.276Z 25 500-519 49 49 49 74 74 74 T F T <---
75622023-09-22T23:27:09.279Z 26 520-539 54 54 54 81 81 81 T F T <---
75632023-09-22T23:27:09.285Z 27 540-559 55 55 55 83 83 83 F F F
75642023-09-22T23:27:09.288Z 28 560-579 53 53 53 80 80 80 F F F
75652023-09-22T23:27:09.293Z 29 580-599 56 56 56 84 84 84 F F F
75662023-09-22T23:27:09.298ZMax gen: 56, Max flush: 84
75672023-09-22T23:27:09.302ZA Difference in extent metadata is expected here
75682023-09-22T23:27:09.309ZOn loop 28
75692023-09-22T23:27:09.312Z
75702023-09-22T23:27:09.316ZVerifying data now
75712023-09-22T23:27:13.976ZNo change after verify
75722023-09-22T23:27:13.980ZLoop: 28 Downstairs dump after verify (and repair):
75732023-09-22T23:27:13.994Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:13.993643359Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1392}
75742023-09-22T23:27:13.997Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:13.993994216Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1392}
75752023-09-22T23:27:14.001Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:13.99401945Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1392}
75762023-09-22T23:27:14.013Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.012527866Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1392}
75772023-09-22T23:27:14.425Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.012581959Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1392}
75782023-09-22T23:27:14.428Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.012603212Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1392}
75792023-09-22T23:27:14.431Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.02340301Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1392}
75802023-09-22T23:27:14.434Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.023456042Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1392}
75812023-09-22T23:27:14.437Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.023494748Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1392}
75822023-09-22T23:27:14.443ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
75832023-09-22T23:27:14.447Z 0 000-019 43 43 43 67 67 67 F F F
75842023-09-22T23:27:14.450Z 1 020-039 54 54 54 82 82 82 F F F
75852023-09-22T23:27:14.454Z 2 040-059 57 57 57 85 85 85 F F F
75862023-09-22T23:27:14.457Z 3 060-079 57 57 57 85 85 85 F F F
75872023-09-22T23:27:14.461Z 4 080-099 57 57 57 85 85 85 F F F
75882023-09-22T23:27:14.464Z 5 100-119 57 57 57 85 85 85 F F F
75892023-09-22T23:27:14.466Z 6 120-139 57 57 57 85 85 85 F F F
75902023-09-22T23:27:14.470Z 7 140-159 57 57 57 85 85 85 F F F
75912023-09-22T23:27:14.473Z 8 160-179 57 57 57 85 85 85 F F F
75922023-09-22T23:27:14.477Z 9 180-199 57 57 57 85 85 85 F F F
75932023-09-22T23:27:14.480Z 10 200-219 57 57 57 85 85 85 F F F
75942023-09-22T23:27:14.483Z 11 220-239 55 55 55 83 83 83 F F F
75952023-09-22T23:27:14.486Z 12 240-259 42 42 42 64 64 64 F F F
75962023-09-22T23:27:14.489Z 13 260-279 57 57 57 85 85 85 F F F
75972023-09-22T23:27:14.492Z 14 280-299 57 57 57 85 85 85 F F F
75982023-09-22T23:27:14.497Z 15 300-319 52 52 52 79 79 79 F F F
75992023-09-22T23:27:14.500Z 16 320-339 57 57 57 85 85 85 F F F
76002023-09-22T23:27:14.503Z 17 340-359 54 54 54 81 81 81 F F F
76012023-09-22T23:27:14.506Z 18 360-379 54 54 54 81 81 81 F F F
76022023-09-22T23:27:14.509Z 19 380-399 51 51 51 76 76 76 F F F
76032023-09-22T23:27:14.512Z 20 400-419 51 51 51 76 76 76 F F F
76042023-09-22T23:27:14.514Z 21 420-439 49 49 49 74 74 74 F F F
76052023-09-22T23:27:14.517Z 22 440-459 53 53 53 80 80 80 F F F
76062023-09-22T23:27:14.520Z 23 460-479 54 54 54 81 81 81 F F F
76072023-09-22T23:27:14.525Z 24 480-499 57 57 57 85 85 85 F F F
76082023-09-22T23:27:14.529Z 25 500-519 57 57 57 85 85 85 F F F
76092023-09-22T23:27:14.532Z 26 520-539 57 57 57 85 85 85 F F F
76102023-09-22T23:27:14.535Z 27 540-559 55 55 55 83 83 83 F F F
76112023-09-22T23:27:14.540Z 28 560-579 53 53 53 80 80 80 F F F
76122023-09-22T23:27:14.543Z 29 580-599 56 56 56 84 84 84 F F F
76132023-09-22T23:27:14.546ZMax gen: 57, Max flush: 85
76142023-09-22T23:27:14.549Z
76152023-09-22T23:27:14.553ZBegin loop 29
76162023-09-22T23:27:14.556ZDownstairs to restart: 2
76172023-09-22T23:27:14.559Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.074113127Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76182023-09-22T23:27:14.561Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.074519482Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76192023-09-22T23:27:14.564Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.07457179Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76202023-09-22T23:27:14.567Z{"msg":"Crucible stats registered with UUID: 5debb1ce-60a3-4f7e-b3fb-f319f61effed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.074597383Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76212023-09-22T23:27:14.571Z{"msg":"Crucible 5debb1ce-60a3-4f7e-b3fb-f319f61effed has session id: 160520d4-8613-40e6-910c-20523f386529","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.074657609Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76222023-09-22T23:27:14.574ZCrucible runtime is spawned
76232023-09-22T23:27:14.577ZThe guest has requested activation with gen:58
76242023-09-22T23:27:14.580Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.074993581Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394,"task":"up_listen"}
76252023-09-22T23:27:14.582Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.075071311Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76262023-09-22T23:27:14.586Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.075138477Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76272023-09-22T23:27:14.589Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.075201287Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394,"looper":"0"}
76282023-09-22T23:27:14.592Z{"msg":"[0] 5debb1ce-60a3-4f7e-b3fb-f319f61effed looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.075457321Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394,"looper":"0"}
76292023-09-22T23:27:14.595Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.07555784Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394,"looper":"2"}
76302023-09-22T23:27:14.598Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.075677625Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394,"looper":"1"}
76312023-09-22T23:27:14.601Z{"msg":"[1] 5debb1ce-60a3-4f7e-b3fb-f319f61effed looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.075889265Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394,"looper":"1"}
76322023-09-22T23:27:14.603Z{"msg":"5debb1ce-60a3-4f7e-b3fb-f319f61effed active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.076153866Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76332023-09-22T23:27:14.607Z{"msg":"Set desired generation to :58","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.0762788Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76342023-09-22T23:27:14.610Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.076344464Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76352023-09-22T23:27:14.613Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.076632605Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76362023-09-22T23:27:14.616Z{"msg":"[0] 5debb1ce-60a3-4f7e-b3fb-f319f61effed (29e94f4f-b777-4647-b54c-169fa73bfbd7) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.077176569Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76372023-09-22T23:27:14.619Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.077247456Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76382023-09-22T23:27:14.622Z{"msg":"[1] 5debb1ce-60a3-4f7e-b3fb-f319f61effed (29e94f4f-b777-4647-b54c-169fa73bfbd7) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.077302572Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76392023-09-22T23:27:14.625Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.077342175Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76402023-09-22T23:27:14.628Z{"msg":"[0] client is_active_req TRUE, promote! session 29e94f4f-b777-4647-b54c-169fa73bfbd7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.07738248Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76412023-09-22T23:27:14.631Z{"msg":"[1] client is_active_req TRUE, promote! session 29e94f4f-b777-4647-b54c-169fa73bfbd7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.077423199Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76422023-09-22T23:27:14.634Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.078085966Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76432023-09-22T23:27:14.637Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.078168373Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76442023-09-22T23:27:14.640Z{"msg":"5debb1ce-60a3-4f7e-b3fb-f319f61effed WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.07828001Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76452023-09-22T23:27:14.643Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.078381448Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76462023-09-22T23:27:14.646Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.078413093Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76472023-09-22T23:27:14.649Z{"msg":"5debb1ce-60a3-4f7e-b3fb-f319f61effed WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.078432375Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76482023-09-22T23:27:14.652Z{"msg":"[0] 5debb1ce-60a3-4f7e-b3fb-f319f61effed (29e94f4f-b777-4647-b54c-169fa73bfbd7) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.080410377Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76492023-09-22T23:27:14.655Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.080443525Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76502023-09-22T23:27:14.658Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:27:14.080462125Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76512023-09-22T23:27:14.662Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.080496968Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76522023-09-22T23:27:14.666Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.080597835Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76532023-09-22T23:27:14.669Z{"msg":"5debb1ce-60a3-4f7e-b3fb-f319f61effed WaitQuorum WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.080714278Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76542023-09-22T23:27:14.672Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.080754441Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76552023-09-22T23:27:14.675Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.080788892Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76562023-09-22T23:27:14.679Z{"msg":"[1] 5debb1ce-60a3-4f7e-b3fb-f319f61effed (29e94f4f-b777-4647-b54c-169fa73bfbd7) WaitQuorum WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.080812607Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76572023-09-22T23:27:14.682Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.080830843Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76582023-09-22T23:27:14.685Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:27:14.080848154Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76592023-09-22T23:27:14.689Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.080865551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76602023-09-22T23:27:14.693Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.080883852Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76612023-09-22T23:27:14.696Z{"msg":"5debb1ce-60a3-4f7e-b3fb-f319f61effed WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.080902155Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76622023-09-22T23:27:14.699Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.080919626Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76632023-09-22T23:27:14.702Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:14.080936719Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76642023-09-22T23:27:15.078Z{"msg":"[2] 5debb1ce-60a3-4f7e-b3fb-f319f61effed looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.076706656Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394,"looper":"2"}
76652023-09-22T23:27:15.081Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.076787601Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76662023-09-22T23:27:15.084Z{"msg":"[2] 5debb1ce-60a3-4f7e-b3fb-f319f61effed (29e94f4f-b777-4647-b54c-169fa73bfbd7) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.077975974Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76672023-09-22T23:27:15.086Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.078029194Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76682023-09-22T23:27:15.090Z{"msg":"[2] client is_active_req TRUE, promote! session 29e94f4f-b777-4647-b54c-169fa73bfbd7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.078073913Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76692023-09-22T23:27:15.093Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.078666121Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76702023-09-22T23:27:15.096Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.07872783Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76712023-09-22T23:27:15.100Z{"msg":"5debb1ce-60a3-4f7e-b3fb-f319f61effed WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.078770639Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76722023-09-22T23:27:15.103Z{"msg":"[2] 5debb1ce-60a3-4f7e-b3fb-f319f61effed (29e94f4f-b777-4647-b54c-169fa73bfbd7) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.081142389Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76732023-09-22T23:27:15.106Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.081230041Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76742023-09-22T23:27:15.109Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:27:15.081278763Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76752023-09-22T23:27:15.112Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.081318473Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76762023-09-22T23:27:15.115Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.081373705Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76772023-09-22T23:27:15.118Z{"msg":"5debb1ce-60a3-4f7e-b3fb-f319f61effed WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.08143176Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76782023-09-22T23:27:15.121Z{"msg":"[0]R flush_numbers[0..12]: [67, 82, 85, 85, 85, 85, 85, 85, 85, 85, 85, 83]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.081476399Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76792023-09-22T23:27:15.124Z{"msg":"[0]R generation[0..12]: [43, 54, 57, 57, 57, 57, 57, 57, 57, 57, 57, 55]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.081517093Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76802023-09-22T23:27:15.127Z{"msg":"The guest has finished waiting for activation with:58
76812023-09-22T23:27:15.131Z[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
76822023-09-22T23:27:15.134Z,"time":"2023-09-22T23:27:15.081559835Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76832023-09-22T23:27:15.137Z{"msg":"[1]R flush_numbers[0..12]: [67, 82, 85, 85, 85, 85, 85, 85, 85, 85, 85, 83]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.081618178Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76842023-09-22T23:27:15.141Z{"msg":"[1]R generation[0..12]: [43, 54, 57, 57, 57, 57, 57, 57, 57, 57, 57, 55]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.081686623Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76852023-09-22T23:27:15.145Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.08176467Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76862023-09-22T23:27:15.148Z{"msg":"[2]R flush_numbers[0..12]: [67, 82, 85, 85, 85, 85, 85, 85, 85, 85, 85, 83]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.081845666Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76872023-09-22T23:27:15.151Z{"msg":"[2]R generation[0..12]: [43, 54, 57, 57, 57, 57, 57, 57, 57, 57, 57, 55]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.081884641Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76882023-09-22T23:27:15.153Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.081918345Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76892023-09-22T23:27:15.159Z{"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.081976703Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76902023-09-22T23:27:15.183Z{"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.082024972Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76912023-09-22T23:27:15.186Z{"msg":"Next flush: 86","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.082105696Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76922023-09-22T23:27:15.189Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.082153809Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76932023-09-22T23:27:15.193Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.082175741Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76942023-09-22T23:27:15.195Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.082261863Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76952023-09-22T23:27:15.199Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.082284574Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76962023-09-22T23:27:15.203Z{"msg":"5debb1ce-60a3-4f7e-b3fb-f319f61effed is now active with session: 29e94f4f-b777-4647-b54c-169fa73bfbd7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.082328199Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76972023-09-22T23:27:15.206Z{"msg":"5debb1ce-60a3-4f7e-b3fb-f319f61effed Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.082370665Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76982023-09-22T23:27:15.209Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.082409294Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
76992023-09-22T23:27:15.212Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.08245337Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
77002023-09-22T23:27:15.215Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.082491722Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
77012023-09-22T23:27:15.218Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.082530368Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
77022023-09-22T23:27:15.222Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.082572931Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
77032023-09-22T23:27:15.227Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.082610675Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
77042023-09-22T23:27:15.230Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.082647802Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
77052023-09-22T23:27:15.233Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.08269332Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
77062023-09-22T23:27:15.236Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.08274265Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
77072023-09-22T23:27:15.239Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
77082023-09-22T23:27:15.242Z,"time":"2023-09-22T23:27:15.082798618Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
77092023-09-22T23:27:15.245Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.082895801Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1394}
77102023-09-22T23:27:15.248ZLoading write count information from file "/tmp/test_repair_verify.data"
77112023-09-22T23:27:15.251ZRun Repair workload
77122023-09-22T23:27:15.254Z01/30 Write block 459 len 16384 data: 6 5 6 5
77132023-09-22T23:27:15.256Z02/30 Read block 168 len 8192
77142023-09-22T23:27:15.259Z03/30 Read block 84 len 12288
77152023-09-22T23:27:15.262Z04/30 Read block 68 len 8192
77162023-09-22T23:27:15.265Z05/30 Write block 548 len 8192 data: 8 9
77172023-09-22T23:27:15.270Z06/30 Write block 512 len 40960 data: 6 8 9 8 8 8 8 6 5 3
77182023-09-22T23:27:15.273Z07/30 Read block 177 len 4096
77192023-09-22T23:27:15.275Z08/30 Write block 65 len 24576 data: 12 11 11 12 12 7
77202023-09-22T23:27:15.279Z09/30 Read block 303 len 8192
77212023-09-22T23:27:15.283Z10/30 Write block 222 len 8192 data: 4 5
77222023-09-22T23:27:15.285Z11/30 Read block 593 len 12288
77232023-09-22T23:27:15.290Z12/30 Write block 181 len 24576 data: 5 6 4 4 3 4
77242023-09-22T23:27:15.293Z13/30 Write block 348 len 20480 data: 6 4 3 4 4
77252023-09-22T23:27:15.296Z14/30 Write block 181 len 16384 data: 6 7 5 5
77262023-09-22T23:27:15.299Z15/30 Flush
77272023-09-22T23:27:15.301Z16/30 Write block 583 len 20480 data: 4 5 4 2 2
77282023-09-22T23:27:15.305Z17/30 Write block 95 len 24576 data: 7 7 5 7 6 5
77292023-09-22T23:27:15.308Z18/30 Write block 400 len 20480 data: 8 7 9 8 8
77302023-09-22T23:27:15.311Z19/30 Write block 51 len 8192 data: 6 4
77312023-09-22T23:27:15.314Z20/30 Write block 152 len 40960 data: 2 2 2 2 2 3 6 8 8 8
77322023-09-22T23:27:15.317Z21/30 Write block 81 len 20480 data: 9 8 7 6 6
77332023-09-22T23:27:15.725Z22/30 Read block 7 len 20480
77342023-09-22T23:27:16.135Z23/30 Write block 368 len 40960 data: 4 3 3 3 3 4 5 5 5 6
77352023-09-22T23:27:16.138Z24/30 Flush
77362023-09-22T23:27:16.140Z25/30 Write block 104 len 32768 data: 6 8 8 8 7 7 8 6
77372023-09-22T23:27:16.144Z26/30 Flush
77382023-09-22T23:27:16.153Z27/30 Write block 592 len 8192 data: 8 9
77392023-09-22T23:27:16.155Z28/30 Write block 175 len 24576 data: 6 6 5 6 6 4
77402023-09-22T23:27:16.159Z29/30 Read block 427 len 20480
77412023-09-22T23:27:16.162Z30/30 Write block 0 len 24576 data: 2 3 4 5 4 4
77422023-09-22T23:27:16.165Z----------------------------------------------------------------
77432023-09-22T23:27:16.169Z Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:30
77442023-09-22T23:27:16.172ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
77452023-09-22T23:27:16.176Z 1 Acked 1000 Write 4 Done Done Sent false
77462023-09-22T23:27:16.179Z 2 Acked 1001 Read 2 Done Done Sent false
77472023-09-22T23:27:16.181Z 3 Acked 1002 Read 3 Done Done Sent false
77482023-09-22T23:27:16.595Z 4 Acked 1003 Read 2 Done Done Sent false
77492023-09-22T23:27:16.598Z 5 Acked 1004 Write 2 Done Done Sent false
77502023-09-22T23:27:16.601Z 6 Acked 1005 Write 10 Done Done Sent false
77512023-09-22T23:27:16.604Z 7 Acked 1006 Read 1 Done Done Sent false
77522023-09-22T23:27:16.607Z 8 Acked 1007 Write 6 Done Done Sent false
77532023-09-22T23:27:16.610Z 9 Acked 1008 Read 2 Done Done Sent false
77542023-09-22T23:27:16.613Z 10 Acked 1009 Write 2 Done Done Sent false
77552023-09-22T23:27:16.616Z 11 Acked 1010 Read 3 Done Done Sent false
77562023-09-22T23:27:16.619Z 12 Acked 1011 Write 6 Done Done Sent false
77572023-09-22T23:27:16.622Z 13 Acked 1012 Write 5 Done Done Sent false
77582023-09-22T23:27:16.626Z 14 Acked 1013 Write 4 Done Done Sent false
77592023-09-22T23:27:16.631Z 15 Acked 1014 Flush 0 Done Done Sent false
77602023-09-22T23:27:16.633Z 16 Acked 1015 Write 5 Done Done Sent false
77612023-09-22T23:27:16.636Z 17 Acked 1016 Write 6 Done Done Sent false
77622023-09-22T23:27:16.639Z 18 Acked 1017 Write 5 Done Done Sent false
77632023-09-22T23:27:16.642Z 19 Acked 1018 Write 2 Done Done Sent false
77642023-09-22T23:27:16.645Z 20 Acked 1019 Write 10 Done Done Sent false
77652023-09-22T23:27:16.648Z 21 Acked 1020 Write 5 Done Done Sent false
77662023-09-22T23:27:16.651Z 22 Acked 1021 Read 5 Done Done Sent false
77672023-09-22T23:27:16.657Z 23 Acked 1022 Write 10 Done Done Sent false
77682023-09-22T23:27:16.660Z 24 Acked 1023 Flush 0 Done Done Sent false
77692023-09-22T23:27:16.663Z 25 Acked 1024 Write 8 Done Done Sent false
77702023-09-22T23:27:16.667Z 26 Acked 1025 Flush 0 Done Done Sent false
77712023-09-22T23:27:16.670Z 27 Acked 1026 Write 2 Done Done Sent false
77722023-09-22T23:27:16.674Z 28 Acked 1027 Write 6 Done Done Sent false
77732023-09-22T23:27:16.677Z 29 Acked 1028 Read 5 Done Done Sent false
77742023-09-22T23:27:16.680Z 30 Acked 1029 Write 6 Sent Sent Sent false
77752023-09-22T23:27:16.683Z STATES DS:0 DS:1 DS:2 TOTAL
77762023-09-22T23:27:17.095Z New 0 0 0 0
77772023-09-22T23:27:17.098Z Sent 1 1 30 32
77782023-09-22T23:27:17.505Z Done 29 29 0 58
77792023-09-22T23:27:17.508Z Skipped 0 0 0 0
77802023-09-22T23:27:17.511Z Error 0 0 0 0
77812023-09-22T23:27:17.514ZLast Flush: 1025 1025 0
77822023-09-22T23:27:17.517ZDownstairs last five completed:
77832023-09-22T23:27:17.521ZUpstairs last five completed: 30 29 28 27 26 25
77842023-09-22T23:27:17.525ZWrote out file "/tmp/test_repair_verify.data"
77852023-09-22T23:27:17.935Z
77862023-09-22T23:27:17.938ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
77872023-09-22T23:27:17.941Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.224867882Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1395}
77882023-09-22T23:27:17.944ZError: Difference in extent metadata found!
77892023-09-22T23:27:17.947Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.225203619Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1395}
77902023-09-22T23:27:17.950Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.225226687Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1395}
77912023-09-22T23:27:17.953Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.251412026Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1395}
77922023-09-22T23:27:17.956Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.251444716Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1395}
77932023-09-22T23:27:17.959Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.25146383Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1395}
77942023-09-22T23:27:17.962Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.26882811Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1395}
77952023-09-22T23:27:17.965Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.268859905Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1395}
77962023-09-22T23:27:17.968Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:15.268878918Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1395}
77972023-09-22T23:27:17.971ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
77982023-09-22T23:27:17.974Z 0 000-019 43 43 43 67 67 67 T T F <---
77992023-09-22T23:27:17.977Z 1 020-039 54 54 54 82 82 82 F F F
78002023-09-22T23:27:17.980Z 2 040-059 58 58 57 87 87 85 F F F <---
78012023-09-22T23:27:17.983Z 3 060-079 58 58 57 86 86 85 F F F <---
78022023-09-22T23:27:17.986Z 4 080-099 58 58 57 87 87 85 F F F <---
78032023-09-22T23:27:17.990Z 5 100-119 58 58 57 88 88 85 F F F <---
78042023-09-22T23:27:17.993Z 6 120-139 57 57 57 85 85 85 F F F
78052023-09-22T23:27:17.996Z 7 140-159 58 58 57 87 87 85 F F F <---
78062023-09-22T23:27:17.999Z 8 160-179 58 58 57 87 87 85 T T F <---
78072023-09-22T23:27:18.002Z 9 180-199 58 58 57 86 86 85 T T F <---
78082023-09-22T23:27:18.005Z 10 200-219 57 57 57 85 85 85 F F F
78092023-09-22T23:27:18.008Z 11 220-239 58 58 55 86 86 83 F F F <---
78102023-09-22T23:27:18.011Z 12 240-259 42 42 42 64 64 64 F F F
78112023-09-22T23:27:18.014Z 13 260-279 57 57 57 85 85 85 F F F
78122023-09-22T23:27:18.017Z 14 280-299 57 57 57 85 85 85 F F F
78132023-09-22T23:27:18.022Z 15 300-319 52 52 52 79 79 79 F F F
78142023-09-22T23:27:18.025Z 16 320-339 57 57 57 85 85 85 F F F
78152023-09-22T23:27:18.028Z 17 340-359 58 58 54 86 86 81 F F F <---
78162023-09-22T23:27:18.031Z 18 360-379 58 58 54 87 87 81 F F F <---
78172023-09-22T23:27:18.034Z 19 380-399 51 51 51 76 76 76 F F F
78182023-09-22T23:27:18.037Z 20 400-419 58 58 51 87 87 76 F F F <---
78192023-09-22T23:27:18.040Z 21 420-439 49 49 49 74 74 74 F F F
78202023-09-22T23:27:18.043Z 22 440-459 58 58 53 86 86 80 F F F <---
78212023-09-22T23:27:18.046Z 23 460-479 58 58 54 86 86 81 F F F <---
78222023-09-22T23:27:18.049Z 24 480-499 57 57 57 85 85 85 F F F
78232023-09-22T23:27:18.052Z 25 500-519 58 58 57 86 86 85 F F F <---
78242023-09-22T23:27:18.055Z 26 520-539 58 58 57 86 86 85 F F F <---
78252023-09-22T23:27:18.058Z 27 540-559 58 58 55 86 86 83 F F F <---
78262023-09-22T23:27:18.061Z 28 560-579 53 53 53 80 80 80 F F F
78272023-09-22T23:27:18.064Z 29 580-599 58 58 56 87 87 84 T T F <---
78282023-09-22T23:27:18.071ZMax gen: 58, Max flush: 88
78292023-09-22T23:27:18.074ZA Difference in extent metadata is expected here
78302023-09-22T23:27:18.077ZOn loop 29
78312023-09-22T23:27:18.080Z
78322023-09-22T23:27:18.083ZVerifying data now
78332023-09-22T23:27:22.117ZNo change after verify
78342023-09-22T23:27:22.120ZLoop: 29 Downstairs dump after verify (and repair):
78352023-09-22T23:27:22.136Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.134760503Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1400}
78362023-09-22T23:27:22.139Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.135185057Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1400}
78372023-09-22T23:27:22.142Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.135244813Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1400}
78382023-09-22T23:27:22.154Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.153940971Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1400}
78392023-09-22T23:27:22.158Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.153993659Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1400}
78402023-09-22T23:27:22.161Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.154013949Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1400}
78412023-09-22T23:27:22.169Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.16486408Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1400}
78422023-09-22T23:27:22.174Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.164915876Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1400}
78432023-09-22T23:27:22.176Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.164936517Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1400}
78442023-09-22T23:27:22.179ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
78452023-09-22T23:27:22.182Z 0 000-019 59 59 59 89 89 89 F F F
78462023-09-22T23:27:22.185Z 1 020-039 54 54 54 82 82 82 F F F
78472023-09-22T23:27:22.187Z 2 040-059 58 58 58 87 87 87 F F F
78482023-09-22T23:27:22.190Z 3 060-079 58 58 58 86 86 86 F F F
78492023-09-22T23:27:22.193Z 4 080-099 58 58 58 87 87 87 F F F
78502023-09-22T23:27:22.196Z 5 100-119 58 58 58 88 88 88 F F F
78512023-09-22T23:27:22.199Z 6 120-139 57 57 57 85 85 85 F F F
78522023-09-22T23:27:22.201Z 7 140-159 58 58 58 87 87 87 F F F
78532023-09-22T23:27:22.204Z 8 160-179 59 59 59 89 89 89 F F F
78542023-09-22T23:27:22.207Z 9 180-199 59 59 59 89 89 89 F F F
78552023-09-22T23:27:22.210Z 10 200-219 57 57 57 85 85 85 F F F
78562023-09-22T23:27:22.213Z 11 220-239 58 58 58 86 86 86 F F F
78572023-09-22T23:27:22.220Z 12 240-259 42 42 42 64 64 64 F F F
78582023-09-22T23:27:22.226Z 13 260-279 57 57 57 85 85 85 F F F
78592023-09-22T23:27:22.229Z 14 280-299 57 57 57 85 85 85 F F F
78602023-09-22T23:27:22.232Z 15 300-319 52 52 52 79 79 79 F F F
78612023-09-22T23:27:22.235Z 16 320-339 57 57 57 85 85 85 F F F
78622023-09-22T23:27:22.237Z 17 340-359 58 58 58 86 86 86 F F F
78632023-09-22T23:27:22.240Z 18 360-379 58 58 58 87 87 87 F F F
78642023-09-22T23:27:22.243Z 19 380-399 51 51 51 76 76 76 F F F
78652023-09-22T23:27:22.246Z 20 400-419 58 58 58 87 87 87 F F F
78662023-09-22T23:27:22.249Z 21 420-439 49 49 49 74 74 74 F F F
78672023-09-22T23:27:22.251Z 22 440-459 58 58 58 86 86 86 F F F
78682023-09-22T23:27:22.254Z 23 460-479 58 58 58 86 86 86 F F F
78692023-09-22T23:27:22.261Z 24 480-499 57 57 57 85 85 85 F F F
78702023-09-22T23:27:22.266Z 25 500-519 58 58 58 86 86 86 F F F
78712023-09-22T23:27:22.269Z 26 520-539 58 58 58 86 86 86 F F F
78722023-09-22T23:27:22.273Z 27 540-559 58 58 58 86 86 86 F F F
78732023-09-22T23:27:22.276Z 28 560-579 53 53 53 80 80 80 F F F
78742023-09-22T23:27:22.278Z 29 580-599 59 59 59 89 89 89 F F F
78752023-09-22T23:27:22.281ZMax gen: 59, Max flush: 89
78762023-09-22T23:27:22.284Z
78772023-09-22T23:27:22.287ZBegin loop 30
78782023-09-22T23:27:22.289ZDownstairs to restart: 1
78792023-09-22T23:27:22.292Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.214173776Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
78802023-09-22T23:27:22.295Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.214541591Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
78812023-09-22T23:27:22.304Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.214620767Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
78822023-09-22T23:27:22.308Z{"msg":"Crucible stats registered with UUID: 0536a690-17bc-4f3a-a5f3-bbe350ba970a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.214644204Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
78832023-09-22T23:27:22.331Z{"msg":"Crucible 0536a690-17bc-4f3a-a5f3-bbe350ba970a has session id: 0c1a5793-f00e-45b8-b785-7ace70f25787","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.214699926Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
78842023-09-22T23:27:22.338ZCrucible runtime is spawned
78852023-09-22T23:27:22.341ZThe guest has requested activation with gen:60
78862023-09-22T23:27:22.346Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.214988224Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402,"task":"up_listen"}
78872023-09-22T23:27:22.348Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.215069155Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
78882023-09-22T23:27:22.351Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.215117897Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
78892023-09-22T23:27:22.354Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.215157764Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402,"looper":"1"}
78902023-09-22T23:27:22.357Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.215285572Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402,"looper":"0"}
78912023-09-22T23:27:22.359Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.215540404Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402,"looper":"2"}
78922023-09-22T23:27:22.362Z{"msg":"[0] 0536a690-17bc-4f3a-a5f3-bbe350ba970a looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.215588694Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402,"looper":"0"}
78932023-09-22T23:27:22.366Z{"msg":"[2] 0536a690-17bc-4f3a-a5f3-bbe350ba970a looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.215758875Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402,"looper":"2"}
78942023-09-22T23:27:22.369Z{"msg":"0536a690-17bc-4f3a-a5f3-bbe350ba970a active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.21603735Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
78952023-09-22T23:27:22.372Z{"msg":"Set desired generation to :60","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.216089458Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
78962023-09-22T23:27:22.377Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.216137222Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
78972023-09-22T23:27:22.380Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.216465568Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
78982023-09-22T23:27:22.383Z{"msg":"[0] 0536a690-17bc-4f3a-a5f3-bbe350ba970a (2c4cab3a-d03f-4969-b0cd-e0eabbc5bb50) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.216784303Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
78992023-09-22T23:27:22.386Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.216857628Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79002023-09-22T23:27:22.388Z{"msg":"[0] client is_active_req TRUE, promote! session 2c4cab3a-d03f-4969-b0cd-e0eabbc5bb50","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.216930989Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79012023-09-22T23:27:22.392Z{"msg":"[2] 0536a690-17bc-4f3a-a5f3-bbe350ba970a (2c4cab3a-d03f-4969-b0cd-e0eabbc5bb50) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.217015031Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79022023-09-22T23:27:22.394Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.217087301Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79032023-09-22T23:27:22.397Z{"msg":"[2] client is_active_req TRUE, promote! session 2c4cab3a-d03f-4969-b0cd-e0eabbc5bb50","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.217164551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79042023-09-22T23:27:22.400Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.217431744Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79052023-09-22T23:27:22.403Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.21751224Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79062023-09-22T23:27:22.406Z{"msg":"0536a690-17bc-4f3a-a5f3-bbe350ba970a WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.217582439Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79072023-09-22T23:27:22.409Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.217666734Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79082023-09-22T23:27:22.412Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.217743266Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79092023-09-22T23:27:22.415Z{"msg":"0536a690-17bc-4f3a-a5f3-bbe350ba970a WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.217817539Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79102023-09-22T23:27:22.419Z{"msg":"[0] 0536a690-17bc-4f3a-a5f3-bbe350ba970a (2c4cab3a-d03f-4969-b0cd-e0eabbc5bb50) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.219811683Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79112023-09-22T23:27:22.422Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.219896431Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79122023-09-22T23:27:22.425Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:27:22.219976141Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79132023-09-22T23:27:22.427Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.220028565Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79142023-09-22T23:27:22.431Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.220051045Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79152023-09-22T23:27:22.433Z{"msg":"0536a690-17bc-4f3a-a5f3-bbe350ba970a WaitQuorum New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.22008912Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79162023-09-22T23:27:22.436Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.220128292Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79172023-09-22T23:27:22.439Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.220166162Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79182023-09-22T23:27:22.442Z{"msg":"[2] 0536a690-17bc-4f3a-a5f3-bbe350ba970a (2c4cab3a-d03f-4969-b0cd-e0eabbc5bb50) WaitQuorum New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.220221992Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79192023-09-22T23:27:22.445Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.220272954Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79202023-09-22T23:27:22.448Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:27:22.220313339Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79212023-09-22T23:27:22.450Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.220351487Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79222023-09-22T23:27:22.453Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.22038954Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79232023-09-22T23:27:22.459Z{"msg":"0536a690-17bc-4f3a-a5f3-bbe350ba970a WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.220427949Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79242023-09-22T23:27:22.462Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.220448124Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79252023-09-22T23:27:22.474Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:22.220465695Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79262023-09-22T23:27:23.217Z{"msg":"[1] 0536a690-17bc-4f3a-a5f3-bbe350ba970a looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.216202552Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402,"looper":"1"}
79272023-09-22T23:27:23.220Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.216288099Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79282023-09-22T23:27:23.234Z{"msg":"[1] 0536a690-17bc-4f3a-a5f3-bbe350ba970a (2c4cab3a-d03f-4969-b0cd-e0eabbc5bb50) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.217426762Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79292023-09-22T23:27:23.237Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.217481108Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79302023-09-22T23:27:23.240Z{"msg":"[1] client is_active_req TRUE, promote! session 2c4cab3a-d03f-4969-b0cd-e0eabbc5bb50","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.217523995Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79312023-09-22T23:27:23.243Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.218240877Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79322023-09-22T23:27:23.246Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.218296226Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79332023-09-22T23:27:23.250Z{"msg":"0536a690-17bc-4f3a-a5f3-bbe350ba970a WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.218338605Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79342023-09-22T23:27:23.253Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.220743191Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79352023-09-22T23:27:23.256Z{"msg":"[1] 0536a690-17bc-4f3a-a5f3-bbe350ba970a (2c4cab3a-d03f-4969-b0cd-e0eabbc5bb50) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.220919506Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79362023-09-22T23:27:23.260Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.220989811Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79372023-09-22T23:27:23.263Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:27:23.221034998Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79382023-09-22T23:27:23.266Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.221055302Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79392023-09-22T23:27:23.270Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.221092679Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79402023-09-22T23:27:23.274Z{"msg":"0536a690-17bc-4f3a-a5f3-bbe350ba970a WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.221115232Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79412023-09-22T23:27:23.277Z{"msg":"[0]R flush_numbers[0..12]: [89, 82, 87, 86, 87, 88, 85, 87, 89, 89, 85, 86]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.221153102Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79422023-09-22T23:27:23.280Z{"msg":"[0]R generation[0..12]: [59, 54, 58, 58, 58, 58, 57, 58, 59, 59, 57, 58]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.221193273Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79432023-09-22T23:27:23.283Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.221234218Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79442023-09-22T23:27:23.286Z{"msg":"[1]R flush_numbers[0..12]: [89, 82, 87, 86, 87, 88, 85, 87, 89, 89, 85, 86]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.2212737Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79452023-09-22T23:27:23.289Z{"msg":"[1]R generation[0..12]: [59, 54, 58, 58, 58, 58, 57, 58, 59, 59, 57, 58]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.22132413Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79462023-09-22T23:27:23.292Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.22134903Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79472023-09-22T23:27:23.295Z{"msg":"[2]R flush_numbers[0..12]: [89, 82, 87, 86, 87, 88, 85, 87, 89, 89, 85, 86]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:60
79482023-09-22T23:27:23.298ZWait for a query_work_queue command to finish before sending IO
79492023-09-22T23:27:23.301Z,"time":"2023-09-22T23:27:23.221414141Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79502023-09-22T23:27:23.304Z{"msg":"[2]R generation[0..12]: [59, 54, 58, 58, 58, 58, 57, 58, 59, 59, 57, 58]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.221650881Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79512023-09-22T23:27:23.307Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.221730925Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79522023-09-22T23:27:23.310Z{"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.221810126Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79532023-09-22T23:27:23.313Z{"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.221878664Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79542023-09-22T23:27:23.316Z{"msg":"Next flush: 90","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.221943953Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79552023-09-22T23:27:23.319Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.222008942Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79562023-09-22T23:27:23.323Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.22207391Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79572023-09-22T23:27:23.326Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.222144102Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79582023-09-22T23:27:23.329Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
79592023-09-22T23:27:23.353Z,"time":"2023-09-22T23:27:23.222214704Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79602023-09-22T23:27:23.356Z{"msg":"0536a690-17bc-4f3a-a5f3-bbe350ba970a is now active with session: 2c4cab3a-d03f-4969-b0cd-e0eabbc5bb50","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.222314184Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79612023-09-22T23:27:23.359Z{"msg":"0536a690-17bc-4f3a-a5f3-bbe350ba970a Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.222364265Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79622023-09-22T23:27:23.362Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.222404286Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79632023-09-22T23:27:23.365Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.222441723Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79642023-09-22T23:27:23.368Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.222479356Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79652023-09-22T23:27:23.371Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.222516946Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79662023-09-22T23:27:23.374Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.222554543Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79672023-09-22T23:27:23.378Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.222591914Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79682023-09-22T23:27:23.381Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.222629321Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79692023-09-22T23:27:23.384Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.222667455Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79702023-09-22T23:27:23.387Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.222704644Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79712023-09-22T23:27:23.390Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.222741615Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79722023-09-22T23:27:23.392Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.222778703Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1402}
79732023-09-22T23:27:23.395ZLoading write count information from file "/tmp/test_repair_verify.data"
79742023-09-22T23:27:23.398ZRun Repair workload
79752023-09-22T23:27:23.401Z01/30 Write block 117 len 28672 data: 4 3 3 5 5 6 6
79762023-09-22T23:27:23.404Z02/30 Write block 7 len 8192 data: 5 5
79772023-09-22T23:27:23.407Z03/30 Write block 72 len 24576 data: 6 3 3 5 6 7
79782023-09-22T23:27:23.410Z04/30 Read block 472 len 24576
79792023-09-22T23:27:23.413Z05/30 Write block 499 len 28672 data: 6 7 6 7 6 6 6
79802023-09-22T23:27:23.416Z06/30 Read block 235 len 24576
79812023-09-22T23:27:23.419Z07/30 Read block 328 len 16384
79822023-09-22T23:27:23.422Z08/30 Read block 163 len 8192
79832023-09-22T23:27:23.425Z09/30 Read block 501 len 20480
79842023-09-22T23:27:23.428Z10/30 Write block 10 len 28672 data: 4 4 4 5 6 6 5
79852023-09-22T23:27:23.431Z11/30 Write block 27 len 24576 data: 5 4 3 3 4 3
79862023-09-22T23:27:23.434Z12/30 Read block 340 len 4096
79872023-09-22T23:27:23.437Z13/30 Write block 6 len 32768 data: 4 6 6 5 5 5 5 6
79882023-09-22T23:27:23.440Z14/30 Write block 60 len 40960 data: 8 8 8 11 10 13 12 12 13 13
79892023-09-22T23:27:23.443Z15/30 Read block 337 len 40960
79902023-09-22T23:27:23.446Z16/30 Read block 424 len 16384
79912023-09-22T23:27:23.449Z17/30 Read block 464 len 36864
79922023-09-22T23:27:23.452Z18/30 Write block 539 len 4096 data: 8
79932023-09-22T23:27:23.454Z19/30 Write block 504 len 4096 data: 7
79942023-09-22T23:27:23.457Z20/30 Write block 162 len 20480 data: 7 6 6 5 4
79952023-09-22T23:27:23.460Z21/30 Flush
79962023-09-22T23:27:23.463Z22/30 Write block 143 len 12288 data: 7 5 5
79972023-09-22T23:27:23.467Z23/30 Write block 522 len 12288 data: 4 5 4
79982023-09-22T23:27:23.469Z24/30 Write block 99 len 16384 data: 7 6 5 4
79992023-09-22T23:27:23.472Z25/30 Read block 314 len 8192
80002023-09-22T23:27:23.475Z26/30 Write block 513 len 40960 data: 9 10 9 9 9 9 7 6 4 5
80012023-09-22T23:27:23.478Z27/30 Read block 591 len 16384
80022023-09-22T23:27:23.481Z28/30 Read block 595 len 16384
80032023-09-22T23:27:23.484Z29/30 Write block 285 len 16384 data: 8 8 8 8
80042023-09-22T23:27:23.487Z30/30 Read block 263 len 16384
80052023-09-22T23:27:23.490Z----------------------------------------------------------------
80062023-09-22T23:27:23.493Z Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:30
80072023-09-22T23:27:23.496ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
80082023-09-22T23:27:23.500Z 1 Acked 1000 Write 7 Done Done Done false
80092023-09-22T23:27:23.504Z 2 Acked 1001 Write 2 Done Sent Done false
80102023-09-22T23:27:23.506Z 3 Acked 1002 Write 6 Done Sent Done false
80112023-09-22T23:27:23.509Z 4 Acked 1003 Read 6 Done Sent Done false
80122023-09-22T23:27:23.512Z 5 Acked 1004 Write 7 Done Sent Done false
80132023-09-22T23:27:23.515Z 6 Acked 1005 Read 6 Done Sent Done false
80142023-09-22T23:27:23.518Z 7 Acked 1006 Read 4 Done Sent Done false
80152023-09-22T23:27:23.521Z 8 Acked 1007 Read 2 Done Sent Done false
80162023-09-22T23:27:23.524Z 9 Acked 1008 Read 5 Done Sent Done false
80172023-09-22T23:27:23.527Z 10 Acked 1009 Write 7 Done Sent Done false
80182023-09-22T23:27:23.530Z 11 Acked 1010 Write 6 Done Sent Done false
80192023-09-22T23:27:23.533Z 12 Acked 1011 Read 1 Done Sent Done false
80202023-09-22T23:27:23.536Z 13 Acked 1012 Write 8 Done Sent Done false
80212023-09-22T23:27:23.539Z 14 Acked 1013 Write 10 Done Sent Done false
80222023-09-22T23:27:23.541Z 15 Acked 1014 Read 10 Done Sent Done false
80232023-09-22T23:27:23.544Z 16 Acked 1015 Read 4 Done Sent Done false
80242023-09-22T23:27:23.547Z 17 Acked 1016 Read 9 Done Sent Done false
80252023-09-22T23:27:23.550Z 18 Acked 1017 Write 1 Done Sent Done false
80262023-09-22T23:27:23.553Z 19 Acked 1018 Write 1 Done Sent Done false
80272023-09-22T23:27:23.556Z 20 Acked 1019 Write 5 Done Sent Done false
80282023-09-22T23:27:23.559Z 21 Acked 1020 Flush 0 Done Sent Done false
80292023-09-22T23:27:23.562Z 22 Acked 1021 Write 3 Done Sent Done false
80302023-09-22T23:27:23.564Z 23 Acked 1022 Write 3 Done Sent Done false
80312023-09-22T23:27:23.571Z 24 Acked 1023 Write 4 Done Sent Done false
80322023-09-22T23:27:23.575Z 25 Acked 1024 Read 2 Done Sent Done false
80332023-09-22T23:27:23.578Z 26 Acked 1025 Write 10 Done Sent Done false
80342023-09-22T23:27:23.581Z 27 Acked 1026 Read 4 Done Sent Done false
80352023-09-22T23:27:23.584Z 28 Acked 1027 Read 4 Done Sent Done false
80362023-09-22T23:27:23.588Z 29 Acked 1028 Write 4 Done Sent Done false
80372023-09-22T23:27:23.591Z 30 Acked 1029 Read 4 Done Sent Done false
80382023-09-22T23:27:23.594Z STATES DS:0 DS:1 DS:2 TOTAL
80392023-09-22T23:27:23.597Z New 0 0 0 0
80402023-09-22T23:27:23.600Z Sent 0 29 0 29
80412023-09-22T23:27:23.603Z Done 30 1 30 61
80422023-09-22T23:27:23.606Z Skipped 0 0 0 0
80432023-09-22T23:27:23.609Z Error 0 0 0 0
80442023-09-22T23:27:23.612ZLast Flush: 1020 0 1020
80452023-09-22T23:27:23.615ZDownstairs last five completed:
80462023-09-22T23:27:23.618ZUpstairs last five completed: 30 29 28 27 26 25
80472023-09-22T23:27:23.621ZWrote out file "/tmp/test_repair_verify.data"
80482023-09-22T23:27:23.625Z
80492023-09-22T23:27:23.628ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
80502023-09-22T23:27:23.632Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.360076915Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1403}
80512023-09-22T23:27:23.635Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.360400664Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1403}
80522023-09-22T23:27:23.638ZError: Difference in extent metadata found!
80532023-09-22T23:27:23.641Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.360424474Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1403}
80542023-09-22T23:27:23.646Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.389538672Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1403}
80552023-09-22T23:27:23.649Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.389572038Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1403}
80562023-09-22T23:27:23.652Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.389591131Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1403}
80572023-09-22T23:27:23.655Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.413941694Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1403}
80582023-09-22T23:27:23.658Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.413978884Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1403}
80592023-09-22T23:27:23.661Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:23.413998584Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1403}
80602023-09-22T23:27:23.665ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
80612023-09-22T23:27:23.668Z 0 000-019 60 59 60 90 89 90 F F F <---
80622023-09-22T23:27:23.671Z 1 020-039 60 54 60 90 82 90 F F F <---
80632023-09-22T23:27:23.674Z 2 040-059 58 58 58 87 87 87 F F F
80642023-09-22T23:27:23.678Z 3 060-079 60 58 60 90 86 90 F F F <---
80652023-09-22T23:27:23.682Z 4 080-099 58 58 58 87 87 87 T F T <---
80662023-09-22T23:27:23.684Z 5 100-119 60 58 60 90 88 90 T T T <---
80672023-09-22T23:27:23.687Z 6 120-139 60 57 60 90 85 90 F T F <---
80682023-09-22T23:27:23.693Z 7 140-159 58 58 58 87 87 87 T F T <---
80692023-09-22T23:27:23.697Z 8 160-179 60 59 60 90 89 90 F F F <---
80702023-09-22T23:27:23.699Z 9 180-199 59 59 59 89 89 89 F F F
80712023-09-22T23:27:23.702Z 10 200-219 57 57 57 85 85 85 F F F
80722023-09-22T23:27:23.705Z 11 220-239 58 58 58 86 86 86 F F F
80732023-09-22T23:27:23.708Z 12 240-259 42 42 42 64 64 64 F F F
80742023-09-22T23:27:23.711Z 13 260-279 57 57 57 85 85 85 F F F
80752023-09-22T23:27:23.714Z 14 280-299 57 57 57 85 85 85 T F T <---
80762023-09-22T23:27:23.717Z 15 300-319 52 52 52 79 79 79 F F F
80772023-09-22T23:27:23.721Z 16 320-339 57 57 57 85 85 85 F F F
80782023-09-22T23:27:23.725Z 17 340-359 58 58 58 86 86 86 F F F
80792023-09-22T23:27:23.731Z 18 360-379 58 58 58 87 87 87 F F F
80802023-09-22T23:27:23.734Z 19 380-399 51 51 51 76 76 76 F F F
80812023-09-22T23:27:23.737Z 20 400-419 58 58 58 87 87 87 F F F
80822023-09-22T23:27:23.740Z 21 420-439 49 49 49 74 74 74 F F F
80832023-09-22T23:27:23.743Z 22 440-459 58 58 58 86 86 86 F F F
80842023-09-22T23:27:23.747Z 23 460-479 58 58 58 86 86 86 F F F
80852023-09-22T23:27:23.750Z 24 480-499 60 57 60 90 85 90 F F F <---
80862023-09-22T23:27:23.753Z 25 500-519 60 58 60 90 86 90 T F T <---
80872023-09-22T23:27:23.756Z 26 520-539 60 58 60 90 86 90 T F T <---
80882023-09-22T23:27:23.759Z 27 540-559 58 58 58 86 86 86 F F F
80892023-09-22T23:27:23.765Z 28 560-579 53 53 53 80 80 80 F F F
80902023-09-22T23:27:23.768Z 29 580-599 59 59 59 89 89 89 F F F
80912023-09-22T23:27:23.771ZMax gen: 60, Max flush: 90
80922023-09-22T23:27:23.774ZA Difference in extent metadata is expected here
80932023-09-22T23:27:23.777ZOn loop 30
80942023-09-22T23:27:23.780Z
80952023-09-22T23:27:23.783ZVerifying data now
80962023-09-22T23:27:29.776ZNo change after verify
80972023-09-22T23:27:29.779ZLoop: 30 Downstairs dump after verify (and repair):
80982023-09-22T23:27:29.795Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.793628616Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1408}
80992023-09-22T23:27:29.797Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.794038821Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1408}
81002023-09-22T23:27:29.800Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.79406353Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1408}
81012023-09-22T23:27:29.813Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.812642815Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1408}
81022023-09-22T23:27:29.816Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.812696375Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1408}
81032023-09-22T23:27:29.819Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.812717024Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1408}
81042023-09-22T23:27:29.825Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.82363371Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1408}
81052023-09-22T23:27:29.828Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.823685066Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1408}
81062023-09-22T23:27:29.831Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.82370548Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1408}
81072023-09-22T23:27:29.835ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
81082023-09-22T23:27:29.838Z 0 000-019 60 60 60 90 90 90 F F F
81092023-09-22T23:27:29.841Z 1 020-039 60 60 60 90 90 90 F F F
81102023-09-22T23:27:29.844Z 2 040-059 58 58 58 87 87 87 F F F
81112023-09-22T23:27:29.847Z 3 060-079 60 60 60 90 90 90 F F F
81122023-09-22T23:27:29.850Z 4 080-099 61 61 61 91 91 91 F F F
81132023-09-22T23:27:29.853Z 5 100-119 61 61 61 91 91 91 F F F
81142023-09-22T23:27:29.856Z 6 120-139 60 60 60 90 90 90 F F F
81152023-09-22T23:27:29.859Z 7 140-159 61 61 61 91 91 91 F F F
81162023-09-22T23:27:29.864Z 8 160-179 60 60 60 90 90 90 F F F
81172023-09-22T23:27:29.867Z 9 180-199 59 59 59 89 89 89 F F F
81182023-09-22T23:27:29.871Z 10 200-219 57 57 57 85 85 85 F F F
81192023-09-22T23:27:29.875Z 11 220-239 58 58 58 86 86 86 F F F
81202023-09-22T23:27:29.878Z 12 240-259 42 42 42 64 64 64 F F F
81212023-09-22T23:27:29.881Z 13 260-279 57 57 57 85 85 85 F F F
81222023-09-22T23:27:29.884Z 14 280-299 61 61 61 91 91 91 F F F
81232023-09-22T23:27:29.890Z 15 300-319 52 52 52 79 79 79 F F F
81242023-09-22T23:27:29.894Z 16 320-339 57 57 57 85 85 85 F F F
81252023-09-22T23:27:29.898Z 17 340-359 58 58 58 86 86 86 F F F
81262023-09-22T23:27:29.903Z 18 360-379 58 58 58 87 87 87 F F F
81272023-09-22T23:27:29.906Z 19 380-399 51 51 51 76 76 76 F F F
81282023-09-22T23:27:29.909Z 20 400-419 58 58 58 87 87 87 F F F
81292023-09-22T23:27:29.913Z 21 420-439 49 49 49 74 74 74 F F F
81302023-09-22T23:27:29.915Z 22 440-459 58 58 58 86 86 86 F F F
81312023-09-22T23:27:29.918Z 23 460-479 58 58 58 86 86 86 F F F
81322023-09-22T23:27:29.921Z 24 480-499 60 60 60 90 90 90 F F F
81332023-09-22T23:27:29.924Z 25 500-519 61 61 61 91 91 91 F F F
81342023-09-22T23:27:29.927Z 26 520-539 61 61 61 91 91 91 F F F
81352023-09-22T23:27:29.930Z 27 540-559 58 58 58 86 86 86 F F F
81362023-09-22T23:27:29.933Z 28 560-579 53 53 53 80 80 80 F F F
81372023-09-22T23:27:29.936Z 29 580-599 59 59 59 89 89 89 F F F
81382023-09-22T23:27:29.940ZMax gen: 61, Max flush: 91
81392023-09-22T23:27:29.943Z
81402023-09-22T23:27:29.946ZBegin loop 31
81412023-09-22T23:27:29.956ZDownstairs to restart: 0
81422023-09-22T23:27:29.959Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.872369218Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81432023-09-22T23:27:29.962Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.872751433Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81442023-09-22T23:27:29.965Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.872829974Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81452023-09-22T23:27:29.968Z{"msg":"Crucible stats registered with UUID: 70f5328c-f599-477b-bdb0-4926357de755","v":0,"name":"crucible","level":30Crucible runtime is spawned
81462023-09-22T23:27:29.971Z,"time":"2023-09-22T23:27:29.872976249Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81472023-09-22T23:27:29.974Z{"msg":"Crucible 70f5328c-f599-477b-bdb0-4926357de755 has session id: a2cc44d7-d527-4bf4-b71f-d2df2ead7f06","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.873103531Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81482023-09-22T23:27:29.977Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.873146989Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410,"task":"up_listen"}
81492023-09-22T23:27:29.980Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.87322314Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81502023-09-22T23:27:29.984ZThe guest has requested activation with gen:62
81512023-09-22T23:27:29.986Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.87328747Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81522023-09-22T23:27:29.990Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.873344546Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410,"looper":"1"}
81532023-09-22T23:27:29.993Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.87378171Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410,"looper":"0"}
81542023-09-22T23:27:29.996Z{"msg":"[1] 70f5328c-f599-477b-bdb0-4926357de755 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.87382137Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410,"looper":"1"}
81552023-09-22T23:27:29.999Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.87395325Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410,"looper":"2"}
81562023-09-22T23:27:30.003Z{"msg":"[2] 70f5328c-f599-477b-bdb0-4926357de755 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.874182811Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410,"looper":"2"}
81572023-09-22T23:27:30.006Z{"msg":"70f5328c-f599-477b-bdb0-4926357de755 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.874410334Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81582023-09-22T23:27:30.010Z{"msg":"Set desired generation to :62","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.874466195Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81592023-09-22T23:27:30.013Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.874543548Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81602023-09-22T23:27:30.022Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.874891383Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81612023-09-22T23:27:30.025Z{"msg":"[1] 70f5328c-f599-477b-bdb0-4926357de755 (ea735ae7-3471-425c-bcf2-9d4bde433b87) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.875260171Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81622023-09-22T23:27:30.028Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.875369162Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81632023-09-22T23:27:30.031Z{"msg":"[1] client is_active_req TRUE, promote! session ea735ae7-3471-425c-bcf2-9d4bde433b87","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.875459485Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81642023-09-22T23:27:30.038Z{"msg":"[2] 70f5328c-f599-477b-bdb0-4926357de755 (ea735ae7-3471-425c-bcf2-9d4bde433b87) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.875535543Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81652023-09-22T23:27:30.041Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.87560846Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81662023-09-22T23:27:30.044Z{"msg":"[2] client is_active_req TRUE, promote! session ea735ae7-3471-425c-bcf2-9d4bde433b87","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.875682604Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81672023-09-22T23:27:30.047Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.875939859Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81682023-09-22T23:27:30.051Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.876020373Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81692023-09-22T23:27:30.054Z{"msg":"70f5328c-f599-477b-bdb0-4926357de755 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.876096402Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81702023-09-22T23:27:30.057Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.876189638Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81712023-09-22T23:27:30.060Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.876243869Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81722023-09-22T23:27:30.063Z{"msg":"70f5328c-f599-477b-bdb0-4926357de755 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.876295896Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81732023-09-22T23:27:30.066Z{"msg":"[1] 70f5328c-f599-477b-bdb0-4926357de755 (ea735ae7-3471-425c-bcf2-9d4bde433b87) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.878351837Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81742023-09-22T23:27:30.070Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.878410246Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81752023-09-22T23:27:30.073Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:27:29.878452894Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81762023-09-22T23:27:30.076Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.878491537Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81772023-09-22T23:27:30.078Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.87853153Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81782023-09-22T23:27:30.082Z{"msg":"70f5328c-f599-477b-bdb0-4926357de755 New WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.878571696Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81792023-09-22T23:27:30.085Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.878610606Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81802023-09-22T23:27:30.088Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.878648812Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81812023-09-22T23:27:30.092Z{"msg":"[2] 70f5328c-f599-477b-bdb0-4926357de755 (ea735ae7-3471-425c-bcf2-9d4bde433b87) New WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.87938082Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81822023-09-22T23:27:30.095Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.879436449Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81832023-09-22T23:27:30.098Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:27:29.879477765Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81842023-09-22T23:27:30.101Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.87951613Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81852023-09-22T23:27:30.104Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.879554818Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81862023-09-22T23:27:30.107Z{"msg":"70f5328c-f599-477b-bdb0-4926357de755 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.879593322Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81872023-09-22T23:27:30.110Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.879613375Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81882023-09-22T23:27:30.113Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:29.879630721Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81892023-09-22T23:27:30.876Z{"msg":"[0] 70f5328c-f599-477b-bdb0-4926357de755 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.875523861Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410,"looper":"0"}
81902023-09-22T23:27:30.879Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.875607774Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81912023-09-22T23:27:30.882Z{"msg":"[0] 70f5328c-f599-477b-bdb0-4926357de755 (ea735ae7-3471-425c-bcf2-9d4bde433b87) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.876640351Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81922023-09-22T23:27:30.885Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.876698028Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81932023-09-22T23:27:30.888Z{"msg":"[0] client is_active_req TRUE, promote! session ea735ae7-3471-425c-bcf2-9d4bde433b87","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.876724777Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81942023-09-22T23:27:30.891Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.877292854Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81952023-09-22T23:27:30.894Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.877354436Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81962023-09-22T23:27:30.897Z{"msg":"70f5328c-f599-477b-bdb0-4926357de755 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.877399916Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81972023-09-22T23:27:30.900Z{"msg":"[0] 70f5328c-f599-477b-bdb0-4926357de755 (ea735ae7-3471-425c-bcf2-9d4bde433b87) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.879710903Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81982023-09-22T23:27:30.903Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.879772151Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
81992023-09-22T23:27:30.906Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:27:30.879835091Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82002023-09-22T23:27:30.909Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.879866539Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82012023-09-22T23:27:30.913Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.879907394Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82022023-09-22T23:27:30.919Z{"msg":"70f5328c-f599-477b-bdb0-4926357de755 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.879947369Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82032023-09-22T23:27:30.923Z{"msg":"[0]R flush_numbers[0..12]: [90, 90, 87, 90, 91, 91, 90, 91, 90, 89, 85, 86]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.87998781Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82042023-09-22T23:27:30.926Z{"msg":"[0]R generation[0..12]: [60, 60, 58, 60, 61, 61, 60, 61, 60, 59, 57, 58]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.880009785Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82052023-09-22T23:27:30.929Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.880031562Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82062023-09-22T23:27:30.935Z{"msg":"[1]R flush_numbers[0..12]: [90, 90, 87, 90, 91, 91, 90, 91, 90, 89, 85, 86]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.88008066Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82072023-09-22T23:27:30.938Z{"msg":"[1]R generation[0..12]: [60, 60, 58, 60, 61, 61, 60, 61, 60, 59, 57, 58]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.880120956Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82082023-09-22T23:27:30.944Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.880143658Z"The guest has finished waiting for activation with:62
82092023-09-22T23:27:30.947ZWait for a query_work_queue command to finish before sending IO
82102023-09-22T23:27:30.950Z,"hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82112023-09-22T23:27:30.953Z{"msg":"[2]R flush_numbers[0..12]: [90, 90, 87, 90, 91, 91, 90, 91, 90, 89, 85, 86]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.880343526Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82122023-09-22T23:27:30.956Z{"msg":"[2]R generation[0..12]: [60, 60, 58, 60, 61, 61, 60, 61, 60, 59, 57, 58]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.880423432Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82132023-09-22T23:27:30.959Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.880447512Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82142023-09-22T23:27:30.962Z{"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.880493818Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82152023-09-22T23:27:30.966Z{"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.880535435Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82162023-09-22T23:27:30.973Z{"msg":"Next flush: 92","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.880588977Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82172023-09-22T23:27:30.976Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.88065744Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82182023-09-22T23:27:30.979Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.880720315Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82192023-09-22T23:27:30.982Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.880763418Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82202023-09-22T23:27:30.985Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.880818564Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82212023-09-22T23:27:30.988Z{"msg":"70f5328c-f599-477b-bdb0-4926357de755 is now active with session: ea735ae7-3471-425c-bcf2-9d4bde433b87","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.880892518Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82222023-09-22T23:27:30.991Z{"msg":"70f5328c-f599-477b-bdb0-4926357de755 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.880943679Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82232023-09-22T23:27:30.995Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.881000563Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82242023-09-22T23:27:30.997Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.88106814Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82252023-09-22T23:27:31.001Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.881134004Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82262023-09-22T23:27:31.004Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.881181946Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82272023-09-22T23:27:31.007Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.88124667Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82282023-09-22T23:27:31.010Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.881309179Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82292023-09-22T23:27:31.013Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.881357921Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82302023-09-22T23:27:31.016ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
82312023-09-22T23:27:31.019Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.881423305Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82322023-09-22T23:27:31.022Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.881488095Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82332023-09-22T23:27:31.025Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.881552976Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82342023-09-22T23:27:31.028Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:30.881615137Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1410}
82352023-09-22T23:27:31.031ZLoading write count information from file "/tmp/test_repair_verify.data"
82362023-09-22T23:27:31.034ZRun Repair workload
82372023-09-22T23:27:31.037Z01/30 Flush
82382023-09-22T23:27:31.041Z02/30 Write block 58 len 28672 data: 6 6 9 9 9 12 11
82392023-09-22T23:27:31.043Z03/30 Write block 43 len 20480 data: 7 7 8 7 8
82402023-09-22T23:27:31.047Z04/30 Read block 463 len 28672
82412023-09-22T23:27:31.049Z05/30 Write block 205 len 28672 data: 7 8 8 10 8 9 10
82422023-09-22T23:27:31.053Z06/30 Read block 302 len 20480
82432023-09-22T23:27:31.057Z07/30 Write block 172 len 24576 data: 7 6 6 7 7 6
82442023-09-22T23:27:31.060Z08/30 Write block 162 len 12288 data: 8 7 7
82452023-09-22T23:27:31.063Z09/30 Read block 391 len 4096
82462023-09-22T23:27:31.067Z10/30 Write block 442 len 32768 data: 4 6 8 8 7 5 7 8
82472023-09-22T23:27:31.070Z11/30 Read block 493 len 16384
82482023-09-22T23:27:31.075Z12/30 Read block 546 len 8192
82492023-09-22T23:27:31.078Z13/30 Write block 190 len 4096 data: 7
82502023-09-22T23:27:31.082Z14/30 Read block 412 len 28672
82512023-09-22T23:27:31.085Z15/30 Read block 370 len 32768
82522023-09-22T23:27:31.088Z16/30 Write block 78 len 12288 data: 6 7 7
82532023-09-22T23:27:31.091Z17/30 Read block 450 len 20480
82542023-09-22T23:27:31.094Z18/30 Read block 73 len 28672
82552023-09-22T23:27:31.101Z19/30 Write block 465 len 24576 data: 6 4 4 4 5 3
82562023-09-22T23:27:31.104Z20/30 Write block 235 len 32768 data: 2 2 2 3 4 4 6 6
82572023-09-22T23:27:31.107Z21/30 Read block 163 len 4096
82582023-09-22T23:27:31.111Z22/30 Read block 533 len 16384
82592023-09-22T23:27:31.114Z23/30 Read block 422 len 32768
82602023-09-22T23:27:31.117Z24/30 Read block 592 len 16384
82612023-09-22T23:27:31.120Z25/30 Write block 101 len 24576 data: 6 5 5 7 9 9
82622023-09-22T23:27:31.123Z26/30 Read block 557 len 40960
82632023-09-22T23:27:31.127Z27/30 Write block 367 len 12288 data: 4 5 4
82642023-09-22T23:27:31.129Z28/30 Write block 79 len 24576 data: 8 8 10 9 8 7
82652023-09-22T23:27:31.132Z29/30 Write block 178 len 28672 data: 7 7 5 7 8 6 6
82662023-09-22T23:27:31.135Z30/30 Write block 562 len 12288 data: 3 3 6
82672023-09-22T23:27:31.138Z----------------------------------------------------------------
82682023-09-22T23:27:31.143Z Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:30
82692023-09-22T23:27:31.146ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
82702023-09-22T23:27:31.149Z 1 Acked 1000 Flush 0 Sent Done Done false
82712023-09-22T23:27:31.153Z 2 Acked 1001 Write 7 Sent Done Done false
82722023-09-22T23:27:31.156Z 3 Acked 1002 Write 5 Sent Done Done false
82732023-09-22T23:27:31.159Z 4 Acked 1003 Read 7 Sent Done Done false
82742023-09-22T23:27:31.162Z 5 Acked 1004 Write 7 Sent Done Done false
82752023-09-22T23:27:31.165Z 6 Acked 1005 Read 5 Sent Done Done false
82762023-09-22T23:27:31.168Z 7 Acked 1006 Write 6 Sent Done Done false
82772023-09-22T23:27:31.171Z 8 Acked 1007 Write 3 Sent Done Done false
82782023-09-22T23:27:31.175Z 9 Acked 1008 Read 1 Sent Done Done false
82792023-09-22T23:27:31.178Z 10 Acked 1009 Write 8 Sent Done Done false
82802023-09-22T23:27:31.182Z 11 Acked 1010 Read 4 Sent Done Done false
82812023-09-22T23:27:31.185Z 12 Acked 1011 Read 2 Sent Done Done false
82822023-09-22T23:27:31.188Z 13 Acked 1012 Write 1 Sent Done Done false
82832023-09-22T23:27:31.191Z 14 Acked 1013 Read 7 Sent Done Done false
82842023-09-22T23:27:31.204Z 15 Acked 1014 Read 8 Sent Done Done false
82852023-09-22T23:27:31.208Z 16 Acked 1015 Write 3 Sent Done Done false
82862023-09-22T23:27:31.211Z 17 Acked 1016 Read 5 Sent Done Done false
82872023-09-22T23:27:31.214Z 18 Acked 1017 Read 7 Sent Done Done false
82882023-09-22T23:27:31.217Z 19 Acked 1018 Write 6 Sent Done Done false
82892023-09-22T23:27:31.220Z 20 Acked 1019 Write 8 Sent Done Done false
82902023-09-22T23:27:31.223Z 21 Acked 1020 Read 1 Sent Done Done false
82912023-09-22T23:27:31.226Z 22 Acked 1021 Read 4 Sent Done Done false
82922023-09-22T23:27:31.229Z 23 Acked 1022 Read 8 Sent Done Done false
82932023-09-22T23:27:31.232Z 24 Acked 1023 Read 4 Sent Done Done false
82942023-09-22T23:27:31.235Z 25 Acked 1024 Write 6 Sent Done Done false
82952023-09-22T23:27:31.238Z 26 Acked 1025 Read 10 Sent Done Done false
82962023-09-22T23:27:31.241Z 27 Acked 1026 Write 3 Sent Done Done false
82972023-09-22T23:27:31.244Z 28 Acked 1027 Write 6 Sent Sent Sent false
82982023-09-22T23:27:31.247Z 29 Acked 1028 Write 7 Sent Sent Sent false
82992023-09-22T23:27:31.251Z 30 Acked 1029 Write 3 Sent Sent Sent false
83002023-09-22T23:27:31.254Z STATES DS:0 DS:1 DS:2 TOTAL
83012023-09-22T23:27:31.257Z New 0 0 0 0
83022023-09-22T23:27:31.260Z Sent 30 3 3 36
83032023-09-22T23:27:31.263Z Done 0 27 27 54
83042023-09-22T23:27:31.267Z Skipped 0 0 0 0
83052023-09-22T23:27:31.270Z Error 0 0 0 0
83062023-09-22T23:27:31.273ZLast Flush: 0 1000 1000
83072023-09-22T23:27:31.276ZDownstairs last five completed:
83082023-09-22T23:27:31.280ZUpstairs last five completed: 30 29 28 27 26 25
83092023-09-22T23:27:31.290ZWrote out file "/tmp/test_repair_verify.data"
83102023-09-22T23:27:31.293Z
83112023-09-22T23:27:31.296ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
83122023-09-22T23:27:31.299Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:31.009298098Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1411}
83132023-09-22T23:27:31.302Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:31.009557495Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1411}
83142023-09-22T23:27:31.305Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:31.009580461Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1411}
83152023-09-22T23:27:31.308Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:31.032292448Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1411}
83162023-09-22T23:27:31.311Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:31.032329576Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1411}
83172023-09-22T23:27:31.314Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:31.032349098Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1411}
83182023-09-22T23:27:31.317Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:31.062505621Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1411}
83192023-09-22T23:27:31.320ZError: Difference in extent metadata found!
83202023-09-22T23:27:31.323Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:31.06253905Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1411}
83212023-09-22T23:27:31.326Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:31.062558512Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1411}
83222023-09-22T23:27:31.329ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83232023-09-22T23:27:31.332Z 0 000-019 60 60 60 90 90 90 F F F
83242023-09-22T23:27:31.335Z 1 020-039 60 60 60 90 90 90 F F F
83252023-09-22T23:27:31.338Z 2 040-059 58 58 58 87 87 87 F T T <---
83262023-09-22T23:27:31.341Z 3 060-079 60 60 60 90 90 90 F T T <---
83272023-09-22T23:27:31.344Z 4 080-099 61 61 61 91 91 91 F T T <---
83282023-09-22T23:27:31.347Z 5 100-119 61 61 61 91 91 91 F T T <---
83292023-09-22T23:27:31.350Z 6 120-139 60 60 60 90 90 90 F F F
83302023-09-22T23:27:31.353Z 7 140-159 61 61 61 91 91 91 F F F
83312023-09-22T23:27:31.356Z 8 160-179 60 60 60 90 90 90 F T T <---
83322023-09-22T23:27:31.359Z 9 180-199 59 59 59 89 89 89 F T T <---
83332023-09-22T23:27:31.363Z 10 200-219 57 57 57 85 85 85 F T T <---
83342023-09-22T23:27:31.366Z 11 220-239 58 58 58 86 86 86 F T T <---
83352023-09-22T23:27:31.372Z 12 240-259 42 42 42 64 64 64 F T T <---
83362023-09-22T23:27:31.378Z 13 260-279 57 57 57 85 85 85 F F F
83372023-09-22T23:27:31.381Z 14 280-299 61 61 61 91 91 91 F F F
83382023-09-22T23:27:31.384Z 15 300-319 52 52 52 79 79 79 F F F
83392023-09-22T23:27:31.387Z 16 320-339 57 57 57 85 85 85 F F F
83402023-09-22T23:27:31.390Z 17 340-359 58 58 58 86 86 86 F F F
83412023-09-22T23:27:31.393Z 18 360-379 58 58 58 87 87 87 F T T <---
83422023-09-22T23:27:31.395Z 19 380-399 51 51 51 76 76 76 F F F
83432023-09-22T23:27:31.398Z 20 400-419 58 58 58 87 87 87 F F F
83442023-09-22T23:27:31.402Z 21 420-439 49 49 49 74 74 74 F F F
83452023-09-22T23:27:31.404Z 22 440-459 58 58 58 86 86 86 F T T <---
83462023-09-22T23:27:31.407Z 23 460-479 58 58 58 86 86 86 F T T <---
83472023-09-22T23:27:31.410Z 24 480-499 60 60 60 90 90 90 F F F
83482023-09-22T23:27:31.413Z 25 500-519 61 61 61 91 91 91 F F F
83492023-09-22T23:27:31.416Z 26 520-539 61 61 61 91 91 91 F F F
83502023-09-22T23:27:31.419Z 27 540-559 58 58 58 86 86 86 F F F
83512023-09-22T23:27:31.422Z 28 560-579 53 53 53 80 80 80 F F F
83522023-09-22T23:27:31.425Z 29 580-599 59 59 59 89 89 89 F F F
83532023-09-22T23:27:31.428ZMax gen: 61, Max flush: 91
83542023-09-22T23:27:31.431ZA Difference in extent metadata is expected here
83552023-09-22T23:27:31.434ZOn loop 31
83562023-09-22T23:27:31.436Z
83572023-09-22T23:27:31.439ZVerifying data now
83582023-09-22T23:27:37.503ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
83592023-09-22T23:27:37.506Z565,567c565,567
83602023-09-22T23:27:37.509Z< 2,
83612023-09-22T23:27:37.512Z< 2,
83622023-09-22T23:27:37.515Z< 5,
83632023-09-22T23:27:37.517Z---
83642023-09-22T23:27:37.521Z> 3,
83652023-09-22T23:27:37.523Z> 3,
83662023-09-22T23:27:37.527Z> 6,
83672023-09-22T23:27:37.531Zdiff found after verify
83682023-09-22T23:27:37.534ZLoop: 31 Downstairs dump after verify (and repair):
83692023-09-22T23:27:37.536Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.528743579Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1419}
83702023-09-22T23:27:37.539Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.529141198Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1419}
83712023-09-22T23:27:37.542Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.529185583Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1419}
83722023-09-22T23:27:37.549Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.548347703Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1419}
83732023-09-22T23:27:37.552Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.548402078Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1419}
83742023-09-22T23:27:37.555Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.548423024Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1419}
83752023-09-22T23:27:37.560Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.559444967Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1419}
83762023-09-22T23:27:37.563Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.55949753Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1419}
83772023-09-22T23:27:37.566Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.55951863Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1419}
83782023-09-22T23:27:37.572ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
83792023-09-22T23:27:37.575Z 0 000-019 60 60 60 90 90 90 F F F
83802023-09-22T23:27:37.578Z 1 020-039 60 60 60 90 90 90 F F F
83812023-09-22T23:27:37.581Z 2 040-059 62 62 62 92 92 92 F F F
83822023-09-22T23:27:37.587Z 3 060-079 62 62 62 92 92 92 F F F
83832023-09-22T23:27:37.590Z 4 080-099 62 62 62 92 92 92 F F F
83842023-09-22T23:27:37.593Z 5 100-119 62 62 62 92 92 92 F F F
83852023-09-22T23:27:37.596Z 6 120-139 60 60 60 90 90 90 F F F
83862023-09-22T23:27:37.600Z 7 140-159 61 61 61 91 91 91 F F F
83872023-09-22T23:27:37.603Z 8 160-179 62 62 62 92 92 92 F F F
83882023-09-22T23:27:37.606Z 9 180-199 62 62 62 92 92 92 F F F
83892023-09-22T23:27:37.611Z 10 200-219 62 62 62 92 92 92 F F F
83902023-09-22T23:27:37.615Z 11 220-239 62 62 62 92 92 92 F F F
83912023-09-22T23:27:37.618Z 12 240-259 62 62 62 92 92 92 F F F
83922023-09-22T23:27:37.621Z 13 260-279 57 57 57 85 85 85 F F F
83932023-09-22T23:27:37.624Z 14 280-299 61 61 61 91 91 91 F F F
83942023-09-22T23:27:37.627Z 15 300-319 52 52 52 79 79 79 F F F
83952023-09-22T23:27:37.630Z 16 320-339 57 57 57 85 85 85 F F F
83962023-09-22T23:27:37.633Z 17 340-359 58 58 58 86 86 86 F F F
83972023-09-22T23:27:37.638Z 18 360-379 62 62 62 92 92 92 F F F
83982023-09-22T23:27:37.663Z 19 380-399 51 51 51 76 76 76 F F F
83992023-09-22T23:27:37.666Z 20 400-419 58 58 58 87 87 87 F F F
84002023-09-22T23:27:37.669Z 21 420-439 49 49 49 74 74 74 F F F
84012023-09-22T23:27:37.677Z 22 440-459 62 62 62 92 92 92 F F F
84022023-09-22T23:27:37.681Z 23 460-479 62 62 62 92 92 92 F F F
84032023-09-22T23:27:37.684Z 24 480-499 60 60 60 90 90 90 F F F
84042023-09-22T23:27:37.688Z 25 500-519 61 61 61 91 91 91 F F F
84052023-09-22T23:27:37.691Z 26 520-539 61 61 61 91 91 91 F F F
84062023-09-22T23:27:37.696Z 27 540-559 58 58 58 86 86 86 F F F
84072023-09-22T23:27:37.699Z 28 560-579 53 53 53 80 80 80 F F F
84082023-09-22T23:27:37.702Z 29 580-599 59 59 59 89 89 89 F F F
84092023-09-22T23:27:37.705ZMax gen: 62, Max flush: 92
84102023-09-22T23:27:37.708Z
84112023-09-22T23:27:37.713ZBegin loop 32
84122023-09-22T23:27:37.716ZDownstairs to restart: 2
84132023-09-22T23:27:37.719Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.613410045Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84142023-09-22T23:27:37.722Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.613813052Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84152023-09-22T23:27:37.725Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.613875541Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84162023-09-22T23:27:37.728Z{"msg":"Crucible stats registered with UUID: 17787aed-ed72-4d83-8d58-d519ddd568d3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.613903812Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84172023-09-22T23:27:37.731Z{"msg":"Crucible 17787aed-ed72-4d83-8d58-d519ddd568d3 has session id: 7aedadda-9f66-4b99-92c5-053efc5e424e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.614095894Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84182023-09-22T23:27:37.734ZCrucible runtime is spawned
84192023-09-22T23:27:37.737ZThe guest has requested activation with gen:64
84202023-09-22T23:27:37.742Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.614356533Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421,"task":"up_listen"}
84212023-09-22T23:27:37.745Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.614411217Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84222023-09-22T23:27:37.748Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.614464088Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84232023-09-22T23:27:37.751Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.61453188Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421,"looper":"1"}
84242023-09-22T23:27:37.754Z{"msg":"[1] 17787aed-ed72-4d83-8d58-d519ddd568d3 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.614795092Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421,"looper":"1"}
84252023-09-22T23:27:37.756Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.614867496Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421,"looper":"0"}
84262023-09-22T23:27:37.759Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.614956489Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421,"looper":"2"}
84272023-09-22T23:27:37.763Z{"msg":"[0] 17787aed-ed72-4d83-8d58-d519ddd568d3 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.615048302Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421,"looper":"0"}
84282023-09-22T23:27:37.765Z{"msg":"17787aed-ed72-4d83-8d58-d519ddd568d3 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.615463515Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84292023-09-22T23:27:37.768Z{"msg":"Set desired generation to :64","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.615517433Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84302023-09-22T23:27:37.775Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.615555395Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84312023-09-22T23:27:37.781Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.615892606Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84322023-09-22T23:27:37.784Z{"msg":"[1] 17787aed-ed72-4d83-8d58-d519ddd568d3 (74650b4a-4963-4d35-bedf-95db492cc246) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.61628821Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84332023-09-22T23:27:37.787Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.616432732Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84342023-09-22T23:27:37.790Z{"msg":"[1] client is_active_req TRUE, promote! session 74650b4a-4963-4d35-bedf-95db492cc246","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.616523243Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84352023-09-22T23:27:37.794Z{"msg":"[0] 17787aed-ed72-4d83-8d58-d519ddd568d3 (74650b4a-4963-4d35-bedf-95db492cc246) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.616568726Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84362023-09-22T23:27:37.799Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.616627926Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84372023-09-22T23:27:37.803Z{"msg":"[0] client is_active_req TRUE, promote! session 74650b4a-4963-4d35-bedf-95db492cc246","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.616676072Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84382023-09-22T23:27:37.806Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.616942478Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84392023-09-22T23:27:37.809Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.61707989Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84402023-09-22T23:27:37.811Z{"msg":"17787aed-ed72-4d83-8d58-d519ddd568d3 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.617126522Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84412023-09-22T23:27:37.814Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.617319523Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84422023-09-22T23:27:37.818Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.617409935Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84432023-09-22T23:27:37.821Z{"msg":"17787aed-ed72-4d83-8d58-d519ddd568d3 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.617443087Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84442023-09-22T23:27:37.824Z{"msg":"[1] 17787aed-ed72-4d83-8d58-d519ddd568d3 (74650b4a-4963-4d35-bedf-95db492cc246) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.619544236Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84452023-09-22T23:27:37.827Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.619593875Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84462023-09-22T23:27:37.830Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:27:37.619663357Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84472023-09-22T23:27:37.833Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.619738583Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84482023-09-22T23:27:37.836Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.619821034Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84492023-09-22T23:27:37.839Z{"msg":"17787aed-ed72-4d83-8d58-d519ddd568d3 WaitActive WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.619907946Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84502023-09-22T23:27:37.842Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.61997872Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84512023-09-22T23:27:37.845Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.620052508Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84522023-09-22T23:27:37.848Z{"msg":"[0] 17787aed-ed72-4d83-8d58-d519ddd568d3 (74650b4a-4963-4d35-bedf-95db492cc246) WaitActive WaitQuorum New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.620119889Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84532023-09-22T23:27:37.852Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.620164503Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84542023-09-22T23:27:37.855Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:27:37.620204105Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84552023-09-22T23:27:37.857Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.620243042Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84562023-09-22T23:27:37.861Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.620264632Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84572023-09-22T23:27:37.863Z{"msg":"17787aed-ed72-4d83-8d58-d519ddd568d3 WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.620382555Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84582023-09-22T23:27:37.866Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.620525504Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84592023-09-22T23:27:37.869Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:37.620559639Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84602023-09-22T23:27:38.636Z{"msg":"[2] 17787aed-ed72-4d83-8d58-d519ddd568d3 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.617192705Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421,"looper":"2"}
84612023-09-22T23:27:38.685Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.617282444Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84622023-09-22T23:27:38.688Z{"msg":"[2] 17787aed-ed72-4d83-8d58-d519ddd568d3 (74650b4a-4963-4d35-bedf-95db492cc246) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.618269579Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84632023-09-22T23:27:38.692Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.618325543Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84642023-09-22T23:27:38.695Z{"msg":"[2] client is_active_req TRUE, promote! session 74650b4a-4963-4d35-bedf-95db492cc246","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.618349196Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84652023-09-22T23:27:38.698Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.618966607Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84662023-09-22T23:27:38.702Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.619026597Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84672023-09-22T23:27:38.705Z{"msg":"17787aed-ed72-4d83-8d58-d519ddd568d3 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.619068287Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84682023-09-22T23:27:38.708Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.621231858Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84692023-09-22T23:27:38.712Z{"msg":"[2] 17787aed-ed72-4d83-8d58-d519ddd568d3 (74650b4a-4963-4d35-bedf-95db492cc246) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.621437909Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84702023-09-22T23:27:38.715Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.621512754Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84712023-09-22T23:27:38.718Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:27:38.621593955Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84722023-09-22T23:27:38.721Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.621649991Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84732023-09-22T23:27:38.724Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.621705039Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84742023-09-22T23:27:38.727Z{"msg":"17787aed-ed72-4d83-8d58-d519ddd568d3 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.621759329Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84752023-09-22T23:27:38.730Z{"msg":"[0]R flush_numbers[0..12]: [90, 90, 92, 92, 92, 92, 90, 91, 92, 92, 92, 92]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.621815098Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84762023-09-22T23:27:38.733Z{"msg":"[0]R generation[0..12]: [60, 60, 62, 62, 62, 62, 60, 61, 62, 62, 62, 62]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:64
84772023-09-22T23:27:38.736ZWait for a query_work_queue command to finish before sending IO
84782023-09-22T23:27:38.739Z,"time":"2023-09-22T23:27:38.621895988Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84792023-09-22T23:27:38.769Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.622166421Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84802023-09-22T23:27:38.802Z{"msg":"[1]R flush_numbers[0..12]: [90, 90, 92, 92, 92, 92, 90, 91, 92, 92, 92, 92]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.622220569Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84812023-09-22T23:27:38.856Z{"msg":"[1]R generation[0..12]: [60, 60, 62, 62, 62, 62, 60, 61, 62, 62, 62, 62]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.622265315Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84822023-09-22T23:27:38.860Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.622307085Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84832023-09-22T23:27:38.863Z{"msg":"[2]R flush_numbers[0..12]: [90, 90, 92, 92, 92, 92, 90, 91, 92, 92, 92, 92]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.622347156Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84842023-09-22T23:27:38.866Z{"msg":"[2]R generation[0..12]: [60, 60, 62, 62, 62, 62, 60, 61, 62, 62, 62, 62]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.622390872Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84852023-09-22T23:27:38.870Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.622435305Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84862023-09-22T23:27:38.873Z{"msg":"Max found gen is 63","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.622473615Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84872023-09-22T23:27:38.876Z{"msg":"Generation requested: 64 >= found:63","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.622512651Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84882023-09-22T23:27:38.879Z{"msg":"Next flush: 93","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.622553486Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84892023-09-22T23:27:38.882Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.622594175Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84902023-09-22T23:27:38.892Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.622632064Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84912023-09-22T23:27:38.895Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.622671377Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84922023-09-22T23:27:38.898Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.622709414Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84932023-09-22T23:27:38.901Z{"msg":"17787aed-ed72-4d83-8d58-d519ddd568d3 is now active with session: 74650b4a-4963-4d35-bedf-95db492cc246","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.622755905Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84942023-09-22T23:27:38.905Z{"msg":"17787aed-ed72-4d83-8d58-d519ddd568d3 Set Active after no repair","v":0,"name":"crucible","levelRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
84952023-09-22T23:27:38.908Z":30,"time":"2023-09-22T23:27:38.622801126Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84962023-09-22T23:27:38.911Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.622840174Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84972023-09-22T23:27:38.915Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.622878112Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84982023-09-22T23:27:38.918Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.622916935Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
84992023-09-22T23:27:38.921Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.622954037Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
85002023-09-22T23:27:38.924Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.622991181Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
85012023-09-22T23:27:38.927Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.623028534Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
85022023-09-22T23:27:38.931Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.623065354Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
85032023-09-22T23:27:38.934Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.623101953Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
85042023-09-22T23:27:38.937Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.623139243Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
85052023-09-22T23:27:38.941Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.623176567Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
85062023-09-22T23:27:38.944Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.623213972Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1421}
85072023-09-22T23:27:38.947ZLoading write count information from file "/tmp/test_repair_verify.data"
85082023-09-22T23:27:38.951ZRun Repair workload
85092023-09-22T23:27:38.954Z01/30 Write block 247 len 4096 data: 3
85102023-09-22T23:27:38.957Z02/30 Write block 351 len 24576 data: 5 5 4 4 5 4
85112023-09-22T23:27:38.960Z03/30 Read block 338 len 8192
85122023-09-22T23:27:38.964Z04/30 Write block 525 len 36864 data: 6 7 7 6 6 4 4 5 5
85132023-09-22T23:27:38.967Z05/30 Read block 286 len 32768
85142023-09-22T23:27:38.970Z06/30 Write block 487 len 24576 data: 5 8 9 8 9 8
85152023-09-22T23:27:38.973Z07/30 Read block 581 len 16384
85162023-09-22T23:27:38.977Z08/30 Write block 434 len 16384 data: 7 6 5 5
85172023-09-22T23:27:38.980Z09/30 Read block 189 len 8192
85182023-09-22T23:27:38.985Z10/30 Write block 237 len 20480 data: 3 4 5 5 7
85192023-09-22T23:27:38.988Z11/30 Write block 173 len 24576 data: 7 7 8 8 7 8
85202023-09-22T23:27:38.991Z12/30 Read block 463 len 4096
85212023-09-22T23:27:38.995Z13/30 Write block 489 len 36864 data: 10 9 10 9 9 8 8 9 8
85222023-09-22T23:27:38.998Z14/30 Read block 308 len 20480
85232023-09-22T23:27:39.001Z15/30 Write block 205 len 8192 data: 8 9
85242023-09-22T23:27:39.007Z16/30 Write block 435 len 36864 data: 7 6 6 5 3 3 5 5 7
85252023-09-22T23:27:39.010Z17/30 Read block 44 len 40960
85262023-09-22T23:27:39.013Z18/30 Read block 586 len 4096
85272023-09-22T23:27:39.016Z19/30 Read block 51 len 36864
85282023-09-22T23:27:39.022Z20/30 Read block 289 len 28672
85292023-09-22T23:27:39.025Z21/30 Write block 533 len 24576 data: 6 5 5 5 8 8
85302023-09-22T23:27:39.029Z22/30 Read block 528 len 32768
85312023-09-22T23:27:39.032Z23/30 Read block 500 len 28672
85322023-09-22T23:27:39.035Z24/30 Read block 96 len 24576
85332023-09-22T23:27:39.039Z25/30 Write block 252 len 28672 data: 6 5 5 4 4 4 4
85342023-09-22T23:27:39.042Z26/30 Read block 485 len 12288
85352023-09-22T23:27:39.046Z27/30 Read block 39 len 36864
85362023-09-22T23:27:39.049Z28/30 Write block 377 len 28672 data: 7 7 6 6 6 5 5
85372023-09-22T23:27:39.052Z29/30 Read block 194 len 36864
85382023-09-22T23:27:39.055Z30/30 Read block 517 len 8192
85392023-09-22T23:27:39.058Z----------------------------------------------------------------
85402023-09-22T23:27:39.062Z Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30
85412023-09-22T23:27:39.065ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
85422023-09-22T23:27:39.068Z 1 Acked 1000 Write 1 Done Done Done false
85432023-09-22T23:27:39.071Z 2 Acked 1001 Write 6 Done Done Done false
85442023-09-22T23:27:39.074Z 3 Acked 1002 Read 2 Done Done Done false
85452023-09-22T23:27:39.077Z 4 Acked 1003 Write 9 Done Done Sent false
85462023-09-22T23:27:39.080Z 5 Acked 1004 Read 8 Done Done Sent false
85472023-09-22T23:27:39.084ZError: Difference in extent metadata found!
85482023-09-22T23:27:39.087Z 6 Acked 1005 Write 6 Done Done Sent false
85492023-09-22T23:27:39.090Z 7 Acked 1006 Read 4 Done Done Sent false
85502023-09-22T23:27:39.093Z 8 Acked 1007 Write 4 Done Done Sent false
85512023-09-22T23:27:39.097Z 9 Acked 1008 Read 2 Done Done Sent false
85522023-09-22T23:27:39.100Z 10 Acked 1009 Write 5 Done Done Sent false
85532023-09-22T23:27:39.103Z 11 Acked 1010 Write 6 Done Done Sent false
85542023-09-22T23:27:39.106Z 12 Acked 1011 Read 1 Done Done Sent false
85552023-09-22T23:27:39.109Z 13 Acked 1012 Write 9 Done Done Sent false
85562023-09-22T23:27:39.112Z 14 Acked 1013 Read 5 Done Done Sent false
85572023-09-22T23:27:39.115Z 15 Acked 1014 Write 2 Done Done Sent false
85582023-09-22T23:27:39.118Z 16 Acked 1015 Write 9 Done Done Sent false
85592023-09-22T23:27:39.121Z 17 Acked 1016 Read 10 Done Done Sent false
85602023-09-22T23:27:39.125Z 18 Acked 1017 Read 1 Done Done Sent false
85612023-09-22T23:27:39.128Z 19 Acked 1018 Read 9 Done Done Sent false
85622023-09-22T23:27:39.131Z 20 Acked 1019 Read 7 Done Done Sent false
85632023-09-22T23:27:39.134Z 21 Acked 1020 Write 6 Done Done Sent false
85642023-09-22T23:27:39.137Z 22 Acked 1021 Read 8 Done Done Sent false
85652023-09-22T23:27:39.140Z 23 Acked 1022 Read 7 Done Done Sent false
85662023-09-22T23:27:39.144Z 24 Acked 1023 Read 6 Done Done Sent false
85672023-09-22T23:27:39.147Z 25 Acked 1024 Write 7 Done Done Sent false
85682023-09-22T23:27:39.150Z 26 Acked 1025 Read 3 Done Done Sent false
85692023-09-22T23:27:39.153Z 27 Acked 1026 Read 9 Done Done Sent false
85702023-09-22T23:27:39.156Z 28 Acked 1027 Write 7 Done Done Sent false
85712023-09-22T23:27:39.159Z 29 Acked 1028 Read 9 Done Done Sent false
85722023-09-22T23:27:39.162Z 30 Acked 1029 Read 2 Done Sent Sent false
85732023-09-22T23:27:39.165Z STATES DS:0 DS:1 DS:2 TOTAL
85742023-09-22T23:27:39.168Z New 0 0 0 0
85752023-09-22T23:27:39.171Z Sent 0 1 27 28
85762023-09-22T23:27:39.174Z Done 30 29 3 62
85772023-09-22T23:27:39.177Z Skipped 0 0 0 0
85782023-09-22T23:27:39.181Z Error 0 0 0 0
85792023-09-22T23:27:39.185ZLast Flush: 0 0 0
85802023-09-22T23:27:39.188ZDownstairs last five completed:
85812023-09-22T23:27:39.191ZUpstairs last five completed: 30 29 28 27 26 25
85822023-09-22T23:27:39.194ZWrote out file "/tmp/test_repair_verify.data"
85832023-09-22T23:27:39.198Z
85842023-09-22T23:27:39.202ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
85852023-09-22T23:27:39.205Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.761498511Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1422}
85862023-09-22T23:27:39.208Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.761822864Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1422}
85872023-09-22T23:27:39.211Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.761846332Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1422}
85882023-09-22T23:27:39.224Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.796742456Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1422}
85892023-09-22T23:27:39.228Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.79677402Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1422}
85902023-09-22T23:27:39.232Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.796793184Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1422}
85912023-09-22T23:27:39.235Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.822199396Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1422}
85922023-09-22T23:27:39.238Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.822229961Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1422}
85932023-09-22T23:27:39.241Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:38.822248525Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1422}
85942023-09-22T23:27:39.244ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
85952023-09-22T23:27:39.247Z 0 000-019 60 60 60 90 90 90 F F F
85962023-09-22T23:27:39.251Z 1 020-039 60 60 60 90 90 90 F F F
85972023-09-22T23:27:39.255Z 2 040-059 62 62 62 92 92 92 F F F
85982023-09-22T23:27:39.258Z 3 060-079 62 62 62 92 92 92 F F F
85992023-09-22T23:27:39.261Z 4 080-099 62 62 62 92 92 92 F F F
86002023-09-22T23:27:39.264Z 5 100-119 62 62 62 92 92 92 F F F
86012023-09-22T23:27:39.267Z 6 120-139 60 60 60 90 90 90 F F F
86022023-09-22T23:27:39.270Z 7 140-159 61 61 61 91 91 91 F F F
86032023-09-22T23:27:39.274Z 8 160-179 62 62 62 92 92 92 T T F <---
86042023-09-22T23:27:39.278Z 9 180-199 62 62 62 92 92 92 F F F
86052023-09-22T23:27:39.281Z 10 200-219 62 62 62 92 92 92 T T F <---
86062023-09-22T23:27:39.284Z 11 220-239 62 62 62 92 92 92 T T F <---
86072023-09-22T23:27:39.287Z 12 240-259 62 62 62 92 92 92 T T T
86082023-09-22T23:27:39.290Z 13 260-279 57 57 57 85 85 85 F F F
86092023-09-22T23:27:39.293Z 14 280-299 61 61 61 91 91 91 F F F
86102023-09-22T23:27:39.297Z 15 300-319 52 52 52 79 79 79 F F F
86112023-09-22T23:27:39.300Z 16 320-339 57 57 57 85 85 85 F F F
86122023-09-22T23:27:39.303Z 17 340-359 58 58 58 86 86 86 T T T
86132023-09-22T23:27:39.307Z 18 360-379 62 62 62 92 92 92 T T F <---
86142023-09-22T23:27:39.310Z 19 380-399 51 51 51 76 76 76 T T F <---
86152023-09-22T23:27:39.312Z 20 400-419 58 58 58 87 87 87 F F F
86162023-09-22T23:27:39.315Z 21 420-439 49 49 49 74 74 74 T T F <---
86172023-09-22T23:27:39.318Z 22 440-459 62 62 62 92 92 92 T T F <---
86182023-09-22T23:27:39.322Z 23 460-479 62 62 62 92 92 92 F F F
86192023-09-22T23:27:39.325Z 24 480-499 60 60 60 90 90 90 T T F <---
86202023-09-22T23:27:39.328Z 25 500-519 61 61 61 91 91 91 F F F
86212023-09-22T23:27:39.331Z 26 520-539 61 61 61 91 91 91 T T F <---
86222023-09-22T23:27:39.334Z 27 540-559 58 58 58 86 86 86 F F F
86232023-09-22T23:27:39.337Z 28 560-579 53 53 53 80 80 80 F F F
86242023-09-22T23:27:39.341Z 29 580-599 59 59 59 89 89 89 F F F
86252023-09-22T23:27:39.343ZMax gen: 62, Max flush: 92
86262023-09-22T23:27:39.346ZA Difference in extent metadata is expected here
86272023-09-22T23:27:39.349ZOn loop 32
86282023-09-22T23:27:39.352Z
86292023-09-22T23:27:39.356ZVerifying data now
86302023-09-22T23:27:45.129ZNo change after verify
86312023-09-22T23:27:45.132ZLoop: 32 Downstairs dump after verify (and repair):
86322023-09-22T23:27:45.149Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.146362574Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1427}
86332023-09-22T23:27:45.153Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.146777792Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1427}
86342023-09-22T23:27:45.157Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.146802953Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1427}
86352023-09-22T23:27:45.166Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.165522523Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1427}
86362023-09-22T23:27:45.169Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.16557594Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1427}
86372023-09-22T23:27:45.172Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.165597009Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1427}
86382023-09-22T23:27:45.177Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.176438277Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1427}
86392023-09-22T23:27:45.181Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.176489709Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1427}
86402023-09-22T23:27:45.184Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.176510301Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1427}
86412023-09-22T23:27:45.188ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
86422023-09-22T23:27:45.191Z 0 000-019 60 60 60 90 90 90 F F F
86432023-09-22T23:27:45.194Z 1 020-039 60 60 60 90 90 90 F F F
86442023-09-22T23:27:45.197Z 2 040-059 62 62 62 92 92 92 F F F
86452023-09-22T23:27:45.200Z 3 060-079 62 62 62 92 92 92 F F F
86462023-09-22T23:27:45.203Z 4 080-099 62 62 62 92 92 92 F F F
86472023-09-22T23:27:45.207Z 5 100-119 62 62 62 92 92 92 F F F
86482023-09-22T23:27:45.210Z 6 120-139 60 60 60 90 90 90 F F F
86492023-09-22T23:27:45.214Z 7 140-159 61 61 61 91 91 91 F F F
86502023-09-22T23:27:45.217Z 8 160-179 63 63 63 93 93 93 F F F
86512023-09-22T23:27:45.221Z 9 180-199 62 62 62 92 92 92 F F F
86522023-09-22T23:27:45.224Z 10 200-219 63 63 63 93 93 93 F F F
86532023-09-22T23:27:45.227Z 11 220-239 63 63 63 93 93 93 F F F
86542023-09-22T23:27:45.230Z 12 240-259 63 63 63 93 93 93 F F F
86552023-09-22T23:27:45.233Z 13 260-279 57 57 57 85 85 85 F F F
86562023-09-22T23:27:45.236Z 14 280-299 61 61 61 91 91 91 F F F
86572023-09-22T23:27:45.241Z 15 300-319 52 52 52 79 79 79 F F F
86582023-09-22T23:27:45.244Z 16 320-339 57 57 57 85 85 85 F F F
86592023-09-22T23:27:45.247Z 17 340-359 63 63 63 93 93 93 F F F
86602023-09-22T23:27:45.250Z 18 360-379 63 63 63 93 93 93 F F F
86612023-09-22T23:27:45.253Z 19 380-399 63 63 63 93 93 93 F F F
86622023-09-22T23:27:45.256Z 20 400-419 58 58 58 87 87 87 F F F
86632023-09-22T23:27:45.259Z 21 420-439 63 63 63 93 93 93 F F F
86642023-09-22T23:27:45.262Z 22 440-459 63 63 63 93 93 93 F F F
86652023-09-22T23:27:45.266Z 23 460-479 62 62 62 92 92 92 F F F
86662023-09-22T23:27:45.269Z 24 480-499 63 63 63 93 93 93 F F F
86672023-09-22T23:27:45.272Z 25 500-519 61 61 61 91 91 91 F F F
86682023-09-22T23:27:45.275Z 26 520-539 63 63 63 93 93 93 F F F
86692023-09-22T23:27:45.279Z 27 540-559 58 58 58 86 86 86 F F F
86702023-09-22T23:27:45.282Z 28 560-579 53 53 53 80 80 80 F F F
86712023-09-22T23:27:45.285Z 29 580-599 59 59 59 89 89 89 F F F
86722023-09-22T23:27:45.288ZMax gen: 63, Max flush: 93
86732023-09-22T23:27:45.291Z
86742023-09-22T23:27:45.294ZBegin loop 33
86752023-09-22T23:27:45.297ZDownstairs to restart: 2
86762023-09-22T23:27:45.300Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.22628795Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
86772023-09-22T23:27:45.303Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.226661784Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
86782023-09-22T23:27:45.306Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.2266923Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
86792023-09-22T23:27:45.309Z{"msg":"Crucible stats registered with UUID: 121ef461-d610-42d2-96ee-cc3ffc6637e5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.226719262Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
86802023-09-22T23:27:45.312Z{"msg":"Crucible 121ef461-d610-42d2-96ee-cc3ffc6637e5 has session id: b147a709-717b-4c7b-9907-3abd1f45bee6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.226740538Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
86812023-09-22T23:27:45.315ZCrucible runtime is spawned
86822023-09-22T23:27:45.318ZThe guest has requested activation with gen:66
86832023-09-22T23:27:45.321Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.227248058Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429,"task":"up_listen"}
86842023-09-22T23:27:45.324Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.22732962Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
86852023-09-22T23:27:45.327Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.227398478Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
86862023-09-22T23:27:45.330Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.227465984Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429,"looper":"1"}
86872023-09-22T23:27:45.333Z{"msg":"[1] 121ef461-d610-42d2-96ee-cc3ffc6637e5 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.227542379Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429,"looper":"1"}
86882023-09-22T23:27:45.336Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.22778245Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429,"looper":"0"}
86892023-09-22T23:27:45.339Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.227886659Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429,"looper":"2"}
86902023-09-22T23:27:45.342Z{"msg":"[0] 121ef461-d610-42d2-96ee-cc3ffc6637e5 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.228035583Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429,"looper":"0"}
86912023-09-22T23:27:45.346Z{"msg":"121ef461-d610-42d2-96ee-cc3ffc6637e5 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.228390456Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
86922023-09-22T23:27:45.349Z{"msg":"Set desired generation to :66","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.228448582Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
86932023-09-22T23:27:45.353Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.228529574Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
86942023-09-22T23:27:45.358Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.228856332Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
86952023-09-22T23:27:45.361Z{"msg":"[1] 121ef461-d610-42d2-96ee-cc3ffc6637e5 (d3c63ff4-2c0a-41f3-9f17-70c880945f86) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.229252583Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
86962023-09-22T23:27:45.366Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.229328275Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
86972023-09-22T23:27:45.369Z{"msg":"[1] client is_active_req TRUE, promote! session d3c63ff4-2c0a-41f3-9f17-70c880945f86","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.229352736Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
86982023-09-22T23:27:45.372Z{"msg":"[0] 121ef461-d610-42d2-96ee-cc3ffc6637e5 (d3c63ff4-2c0a-41f3-9f17-70c880945f86) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.229561636Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
86992023-09-22T23:27:45.375Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.229637526Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87002023-09-22T23:27:45.378Z{"msg":"[0] client is_active_req TRUE, promote! session d3c63ff4-2c0a-41f3-9f17-70c880945f86","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.229694837Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87012023-09-22T23:27:45.381Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.230102819Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87022023-09-22T23:27:45.384Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.230232398Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87032023-09-22T23:27:45.389Z{"msg":"121ef461-d610-42d2-96ee-cc3ffc6637e5 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.2303131Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87042023-09-22T23:27:45.392Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.230373115Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87052023-09-22T23:27:45.395Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.230449378Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87062023-09-22T23:27:45.398Z{"msg":"121ef461-d610-42d2-96ee-cc3ffc6637e5 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.230519236Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87072023-09-22T23:27:45.401Z{"msg":"[0] 121ef461-d610-42d2-96ee-cc3ffc6637e5 (d3c63ff4-2c0a-41f3-9f17-70c880945f86) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.232562041Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87082023-09-22T23:27:45.404Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.232642052Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87092023-09-22T23:27:45.407Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:27:45.232700871Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87102023-09-22T23:27:45.410Z{"msg":"[1] 121ef461-d610-42d2-96ee-cc3ffc6637e5 (d3c63ff4-2c0a-41f3-9f17-70c880945f86) WaitQuorum WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.232759674Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87112023-09-22T23:27:45.414Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.232813646Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87122023-09-22T23:27:45.418Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:27:45.232866942Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87132023-09-22T23:27:45.421Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.232920429Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87142023-09-22T23:27:45.424Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.232973715Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87152023-09-22T23:27:45.427Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.233026963Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87162023-09-22T23:27:45.431Z{"msg":"121ef461-d610-42d2-96ee-cc3ffc6637e5 WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.233094659Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87172023-09-22T23:27:45.434Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.233152357Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87182023-09-22T23:27:45.437Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.233205284Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87192023-09-22T23:27:45.441Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.233276201Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87202023-09-22T23:27:45.444Z{"msg":"121ef461-d610-42d2-96ee-cc3ffc6637e5 WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.233328403Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87212023-09-22T23:27:45.448Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.233381527Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87222023-09-22T23:27:45.451Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:45.23341047Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87232023-09-22T23:27:46.230Z{"msg":"[2] 121ef461-d610-42d2-96ee-cc3ffc6637e5 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.229696203Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429,"looper":"2"}
87242023-09-22T23:27:46.233Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.22978879Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87252023-09-22T23:27:46.237Z{"msg":"[2] 121ef461-d610-42d2-96ee-cc3ffc6637e5 (d3c63ff4-2c0a-41f3-9f17-70c880945f86) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.230780785Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87262023-09-22T23:27:46.240Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.230838225Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87272023-09-22T23:27:46.243Z{"msg":"[2] client is_active_req TRUE, promote! session d3c63ff4-2c0a-41f3-9f17-70c880945f86","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.230882002Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87282023-09-22T23:27:46.247Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.231453324Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87292023-09-22T23:27:46.250Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.231517782Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87302023-09-22T23:27:46.253Z{"msg":"121ef461-d610-42d2-96ee-cc3ffc6637e5 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.231561302Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87312023-09-22T23:27:46.257Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.2340386Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87322023-09-22T23:27:46.260Z{"msg":"[2] 121ef461-d610-42d2-96ee-cc3ffc6637e5 (d3c63ff4-2c0a-41f3-9f17-70c880945f86) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.234110258Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87332023-09-22T23:27:46.264Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.234167584Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87342023-09-22T23:27:46.267Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:27:46.234235945Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87352023-09-22T23:27:46.271Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.234290321Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87362023-09-22T23:27:46.274Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.234345234Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87372023-09-22T23:27:46.278Z{"msg":"121ef461-d610-42d2-96ee-cc3ffc6637e5 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.23440061Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87382023-09-22T23:27:46.280Z{"msg":"[0]R flush_numbers[0..12]: [90, 90, 92, 92, 92, 92, 90, 91, 93, 92, 93, 93]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.234456694Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87392023-09-22T23:27:46.283Z{"msg":"[0]R generation[0..12]: [60, 60, 62, 62, 62, 62, 60, 61, 63, 62, 63, 63]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:66
87402023-09-22T23:27:46.287ZWait for a query_work_queue command to finish before sending IO
87412023-09-22T23:27:46.290Z,"time":"2023-09-22T23:27:46.234513301Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87422023-09-22T23:27:46.294Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.234707738Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87432023-09-22T23:27:46.297Z{"msg":"[1]R flush_numbers[0..12]: [90, 90, 92, 92, 92, 92, 90, 91, 93, 92, 93, 93]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.234754352Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87442023-09-22T23:27:46.300Z{"msg":"[1]R generation[0..12]: [60, 60, 62, 62, 62, 62, 60, 61, 63, 62, 63, 63]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.234807889Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87452023-09-22T23:27:46.304Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.234865332Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87462023-09-22T23:27:46.307Z{"msg":"[2]R flush_numbers[0..12]: [90, 90, 92, 92, 92, 92, 90, 91, 93, 92, 93, 93]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.234920389Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87472023-09-22T23:27:46.310Z{"msg":"[2]R generation[0..12]: [60, 60, 62, 62, 62, 62, 60, 61, 63, 62, 63, 63]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.234976365Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87482023-09-22T23:27:46.313Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.235033663Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87492023-09-22T23:27:46.316Z{"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.235068177Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87502023-09-22T23:27:46.320Z{"msg":"Generation requested: 66 >= found:64","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.235133096Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87512023-09-22T23:27:46.322Z{"msg":"Next flush: 94","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.235186995Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87522023-09-22T23:27:46.326Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.235246056Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87532023-09-22T23:27:46.329Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.235300252Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87542023-09-22T23:27:46.332Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.235354378Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87552023-09-22T23:27:46.335Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.235407925Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87562023-09-22T23:27:46.338Z{"msg":"121ef461-d610-42d2-96ee-cc3ffc6637e5 is now active with session: d3c63ff4-2c0a-41f3-9f17-70c880945f86","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.235464249Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87572023-09-22T23:27:46.341Z{"msg":"121ef461-d610-42d2-96ee-cc3ffc6637e5 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.235523293Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87582023-09-22T23:27:46.344Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
87592023-09-22T23:27:46.348Z,"time":"2023-09-22T23:27:46.235589663Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87602023-09-22T23:27:46.351Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.235698843Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87612023-09-22T23:27:46.354Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.235755621Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87622023-09-22T23:27:46.358Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.235817308Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87632023-09-22T23:27:46.361Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.235870954Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87642023-09-22T23:27:46.364Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.235930085Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87652023-09-22T23:27:46.368Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.235984254Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87662023-09-22T23:27:46.371Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.236037587Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87672023-09-22T23:27:46.375Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.236091395Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87682023-09-22T23:27:46.379Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.236147218Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87692023-09-22T23:27:46.383Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.236202055Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1429}
87702023-09-22T23:27:46.386ZLoading write count information from file "/tmp/test_repair_verify.data"
87712023-09-22T23:27:46.389ZRun Repair workload
87722023-09-22T23:27:46.392Z01/30 Write block 22 len 24576 data: 5 4 5 5 5 6
87732023-09-22T23:27:46.395Z02/30 Read block 517 len 24576
87742023-09-22T23:27:46.399Z03/30 Write block 305 len 24576 data: 8 8 8 7 5 3
87752023-09-22T23:27:46.402Z04/30 Read block 286 len 40960
87762023-09-22T23:27:46.405Z05/30 Write block 566 len 24576 data: 5 8 9 9 6 5
87772023-09-22T23:27:46.409Z06/30 Read block 373 len 20480
87782023-09-22T23:27:46.412Z07/30 Read block 158 len 24576
87792023-09-22T23:27:46.415Z08/30 Write block 328 len 32768 data: 3 2 2 3 3 4 6 7
87802023-09-22T23:27:46.418Z09/30 Read block 323 len 36864
87812023-09-22T23:27:46.421Z10/30 Read block 274 len 40960
87822023-09-22T23:27:46.425Z11/30 Write block 97 len 8192 data: 6 8
87832023-09-22T23:27:46.429Z12/30 Flush
87842023-09-22T23:27:46.432Z13/30 Write block 479 len 32768 data: 7 6 5 6 5 7 7 7
87852023-09-22T23:27:46.435Z14/30 Write block 346 len 16384 data: 7 7 7 5
87862023-09-22T23:27:46.438Z15/30 Flush
87872023-09-22T23:27:46.441Z16/30 Write block 149 len 28672 data: 4 4 4 3 3 3 3
87882023-09-22T23:27:46.864Z17/30 Read block 91 len 8192
87892023-09-22T23:27:46.867Z18/30 Write block 225 len 12288 data: 6 7 5
87902023-09-22T23:27:46.871Z19/30 Read block 380 len 36864
87912023-09-22T23:27:46.874Z20/30 Read block 3 len 12288
87922023-09-22T23:27:46.877Z21/30 Read block 231 len 12288
87932023-09-22T23:27:46.880Z22/30 Read block 541 len 28672
87942023-09-22T23:27:46.883Z23/30 Flush
87952023-09-22T23:27:46.886Z24/30 Write block 414 len 4096 data: 7
87962023-09-22T23:27:46.889Z25/30 Flush
87972023-09-22T23:27:46.892Z26/30 Write block 151 len 24576 data: 5 4 4 4 4 3
87982023-09-22T23:27:46.895Z27/30 Read block 144 len 32768
87992023-09-22T23:27:46.898Z28/30 Write block 207 len 32768 data: 9 11 9 10 11 8 8 6
88002023-09-22T23:27:46.903Z29/30 Read block 28 len 40960
88012023-09-22T23:27:46.906Z30/30 Read block 585 len 20480
88022023-09-22T23:27:46.909Z----------------------------------------------------------------
88032023-09-22T23:27:46.912Z Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30
88042023-09-22T23:27:46.915ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
88052023-09-22T23:27:46.921Z 1 Acked 1000 Write 6 Done Done Done false
88062023-09-22T23:27:46.924Z 2 Acked 1001 Read 6 Done Done Done false
88072023-09-22T23:27:46.927Z 3 Acked 1002 Write 6 Done Done Done false
88082023-09-22T23:27:46.930Z 4 Acked 1003 Read 10 Done Done Sent false
88092023-09-22T23:27:46.933Z 5 Acked 1004 Write 6 Done Done Sent false
88102023-09-22T23:27:46.938Z 6 Acked 1005 Read 5 Done Done Sent false
88112023-09-22T23:27:46.942Z 7 Acked 1006 Read 6 Done Done Sent false
88122023-09-22T23:27:46.945Z 8 Acked 1007 Write 8 Done Done Sent false
88132023-09-22T23:27:46.947Z 9 Acked 1008 Read 9 Done Done Sent false
88142023-09-22T23:27:46.950Z 10 Acked 1009 Read 10 Done Done Sent false
88152023-09-22T23:27:46.953Z 11 Acked 1010 Write 2 Done Done Sent false
88162023-09-22T23:27:46.956Z 12 Acked 1011 Flush 0 Done Done Sent false
88172023-09-22T23:27:46.960Z 13 Acked 1012 Write 8 Done Done Sent false
88182023-09-22T23:27:46.963Z 14 Acked 1013 Write 4 Done Done Sent false
88192023-09-22T23:27:46.966Z 15 Acked 1014 Flush 0 Done Done Sent false
88202023-09-22T23:27:46.971Z 16 Acked 1015 Write 7 Done Done Sent false
88212023-09-22T23:27:46.974Z 17 Acked 1016 Read 2 Done Done Sent false
88222023-09-22T23:27:46.977Z 18 Acked 1017 Write 3 Done Done Sent false
88232023-09-22T23:27:46.980Z 19 Acked 1018 Read 9 Done Done Sent false
88242023-09-22T23:27:46.983Z 20 Acked 1019 Read 3 Done Done Sent false
88252023-09-22T23:27:46.986Z 21 Acked 1020 Read 3 Done Done Sent false
88262023-09-22T23:27:46.989Z 22 Acked 1021 Read 7 Done Done Sent false
88272023-09-22T23:27:46.992Z 23 Acked 1022 Flush 0 Done Done Sent false
88282023-09-22T23:27:46.996Z 24 Acked 1023 Write 1 Done Done Sent false
88292023-09-22T23:27:47.000Z 25 Acked 1024 Flush 0 Done Done Sent false
88302023-09-22T23:27:47.003Z 26 Acked 1025 Write 6 Done Done Sent false
88312023-09-22T23:27:47.007Z 27 Acked 1026 Read 8 Done Done Sent false
88322023-09-22T23:27:47.010Z 28 Acked 1027 Write 8 Done Done Sent false
88332023-09-22T23:27:47.015Z 29 Acked 1028 Read 10 Done Done Sent false
88342023-09-22T23:27:47.019Z 30 Acked 1029 Read 5 Done Sent Sent false
88352023-09-22T23:27:47.023Z STATES DS:0 DS:1 DS:2 TOTAL
88362023-09-22T23:27:47.026Z New 0 0 0 0
88372023-09-22T23:27:47.029Z Sent 0 1 27 28
88382023-09-22T23:27:47.032Z Done 30 29 3 62
88392023-09-22T23:27:47.036Z Skipped 0 0 0 0
88402023-09-22T23:27:47.039Z Error 0 0 0 0
88412023-09-22T23:27:47.043ZLast Flush: 1024 1024 0
88422023-09-22T23:27:47.046ZDownstairs last five completed:
88432023-09-22T23:27:47.049ZUpstairs last five completed: 30 29 28 27 26 25
88442023-09-22T23:27:47.052ZWrote out file "/tmp/test_repair_verify.data"
88452023-09-22T23:27:47.056Z
88462023-09-22T23:27:47.059ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
88472023-09-22T23:27:47.062Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.384416112Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1430}
88482023-09-22T23:27:47.065Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.38471642Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1430}
88492023-09-22T23:27:47.068Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.384739628Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1430}
88502023-09-22T23:27:47.071Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.407685012Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1430}
88512023-09-22T23:27:47.074Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.407717721Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1430}
88522023-09-22T23:27:47.078Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.407737245Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1430}
88532023-09-22T23:27:47.081Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.421465512Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1430}
88542023-09-22T23:27:47.084ZError: Difference in extent metadata found!
88552023-09-22T23:27:47.087Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.421496789Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1430}
88562023-09-22T23:27:47.092Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:46.421515587Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1430}
88572023-09-22T23:27:47.096ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
88582023-09-22T23:27:47.099Z 0 000-019 60 60 60 90 90 90 F F F
88592023-09-22T23:27:47.102Z 1 020-039 66 66 60 94 94 90 F F T <---
88602023-09-22T23:27:47.105Z 2 040-059 62 62 62 92 92 92 F F F
88612023-09-22T23:27:47.111Z 3 060-079 62 62 62 92 92 92 F F F
88622023-09-22T23:27:47.114Z 4 080-099 66 66 62 94 94 92 F F F <---
88632023-09-22T23:27:47.118Z 5 100-119 62 62 62 92 92 92 F F F
88642023-09-22T23:27:47.122Z 6 120-139 60 60 60 90 90 90 F F F
88652023-09-22T23:27:47.126Z 7 140-159 66 66 61 96 96 91 T T F <---
88662023-09-22T23:27:47.128Z 8 160-179 63 63 63 93 93 93 F F F
88672023-09-22T23:27:47.132Z 9 180-199 62 62 62 92 92 92 F F F
88682023-09-22T23:27:47.135Z 10 200-219 63 63 63 93 93 93 T T F <---
88692023-09-22T23:27:47.138Z 11 220-239 66 66 63 96 96 93 F F F <---
88702023-09-22T23:27:47.141Z 12 240-259 63 63 63 93 93 93 F F F
88712023-09-22T23:27:47.144Z 13 260-279 57 57 57 85 85 85 F F F
88722023-09-22T23:27:47.147Z 14 280-299 61 61 61 91 91 91 F F F
88732023-09-22T23:27:47.150Z 15 300-319 66 66 52 94 94 79 F F T <---
88742023-09-22T23:27:47.153Z 16 320-339 66 66 57 94 94 85 F F F <---
88752023-09-22T23:27:47.157Z 17 340-359 66 66 63 95 95 93 F F F <---
88762023-09-22T23:27:47.160Z 18 360-379 63 63 63 93 93 93 F F F
88772023-09-22T23:27:47.162Z 19 380-399 63 63 63 93 93 93 F F F
88782023-09-22T23:27:47.166Z 20 400-419 66 66 58 97 97 87 F F F <---
88792023-09-22T23:27:47.169Z 21 420-439 63 63 63 93 93 93 F F F
88802023-09-22T23:27:47.172Z 22 440-459 63 63 63 93 93 93 F F F
88812023-09-22T23:27:47.175Z 23 460-479 66 66 62 95 95 92 F F F <---
88822023-09-22T23:27:47.178Z 24 480-499 66 66 63 95 95 93 F F F <---
88832023-09-22T23:27:47.181Z 25 500-519 61 61 61 91 91 91 F F F
88842023-09-22T23:27:47.184Z 26 520-539 63 63 63 93 93 93 F F F
88852023-09-22T23:27:47.187Z 27 540-559 58 58 58 86 86 86 F F F
88862023-09-22T23:27:47.190Z 28 560-579 66 66 53 94 94 80 F F F <---
88872023-09-22T23:27:47.194Z 29 580-599 59 59 59 89 89 89 F F F
88882023-09-22T23:27:47.197ZMax gen: 66, Max flush: 97
88892023-09-22T23:27:47.199ZA Difference in extent metadata is expected here
88902023-09-22T23:27:47.203ZOn loop 33
88912023-09-22T23:27:47.205Z
88922023-09-22T23:27:47.209ZVerifying data now
88932023-09-22T23:27:52.756ZNo change after verify
88942023-09-22T23:27:52.759ZLoop: 33 Downstairs dump after verify (and repair):
88952023-09-22T23:27:52.774Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.773624458Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1435}
88962023-09-22T23:27:52.777Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.774034153Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1435}
88972023-09-22T23:27:52.781Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.774074343Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1435}
88982023-09-22T23:27:52.793Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.792609422Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1435}
88992023-09-22T23:27:52.796Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.792665504Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1435}
89002023-09-22T23:27:52.799Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.792686503Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1435}
89012023-09-22T23:27:52.804Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.803490649Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1435}
89022023-09-22T23:27:52.807Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.803548791Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1435}
89032023-09-22T23:27:52.810Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.803613381Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1435}
89042023-09-22T23:27:52.815ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
89052023-09-22T23:27:52.818Z 0 000-019 60 60 60 90 90 90 F F F
89062023-09-22T23:27:52.821Z 1 020-039 66 66 66 94 94 94 F F F
89072023-09-22T23:27:52.824Z 2 040-059 62 62 62 92 92 92 F F F
89082023-09-22T23:27:52.827Z 3 060-079 62 62 62 92 92 92 F F F
89092023-09-22T23:27:52.830Z 4 080-099 66 66 66 94 94 94 F F F
89102023-09-22T23:27:52.834Z 5 100-119 62 62 62 92 92 92 F F F
89112023-09-22T23:27:52.837Z 6 120-139 60 60 60 90 90 90 F F F
89122023-09-22T23:27:52.840Z 7 140-159 67 67 67 98 98 98 F F F
89132023-09-22T23:27:52.843Z 8 160-179 63 63 63 93 93 93 F F F
89142023-09-22T23:27:52.846Z 9 180-199 62 62 62 92 92 92 F F F
89152023-09-22T23:27:52.849Z 10 200-219 67 67 67 98 98 98 F F F
89162023-09-22T23:27:52.852Z 11 220-239 66 66 66 96 96 96 F F F
89172023-09-22T23:27:52.855Z 12 240-259 63 63 63 93 93 93 F F F
89182023-09-22T23:27:52.858Z 13 260-279 57 57 57 85 85 85 F F F
89192023-09-22T23:27:52.861Z 14 280-299 61 61 61 91 91 91 F F F
89202023-09-22T23:27:52.864Z 15 300-319 66 66 66 94 94 94 F F F
89212023-09-22T23:27:52.867Z 16 320-339 66 66 66 94 94 94 F F F
89222023-09-22T23:27:52.870Z 17 340-359 66 66 66 95 95 95 F F F
89232023-09-22T23:27:52.873Z 18 360-379 63 63 63 93 93 93 F F F
89242023-09-22T23:27:52.876Z 19 380-399 63 63 63 93 93 93 F F F
89252023-09-22T23:27:52.879Z 20 400-419 66 66 66 97 97 97 F F F
89262023-09-22T23:27:52.883Z 21 420-439 63 63 63 93 93 93 F F F
89272023-09-22T23:27:52.886Z 22 440-459 63 63 63 93 93 93 F F F
89282023-09-22T23:27:52.889Z 23 460-479 66 66 66 95 95 95 F F F
89292023-09-22T23:27:52.892Z 24 480-499 66 66 66 95 95 95 F F F
89302023-09-22T23:27:52.895Z 25 500-519 61 61 61 91 91 91 F F F
89312023-09-22T23:27:52.899Z 26 520-539 63 63 63 93 93 93 F F F
89322023-09-22T23:27:52.902Z 27 540-559 58 58 58 86 86 86 F F F
89332023-09-22T23:27:52.906Z 28 560-579 66 66 66 94 94 94 F F F
89342023-09-22T23:27:52.909Z 29 580-599 59 59 59 89 89 89 F F F
89352023-09-22T23:27:52.912ZMax gen: 67, Max flush: 98
89362023-09-22T23:27:52.915Z
89372023-09-22T23:27:52.918ZBegin loop 34
89382023-09-22T23:27:52.921ZDownstairs to restart: 1
89392023-09-22T23:27:52.924Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.851837944Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89402023-09-22T23:27:52.927Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.852313613Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89412023-09-22T23:27:52.931Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.852365815Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89422023-09-22T23:27:52.934Z{"msg":"Crucible stats registered with UUID: b84daa34-f037-4da5-9129-454c091306d3","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.852398107Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89432023-09-22T23:27:52.937ZCrucible runtime is spawned
89442023-09-22T23:27:52.940Z{"msg":"Crucible b84daa34-f037-4da5-9129-454c091306d3 has session id: 77857fb8-cda3-47ff-988e-040d43e41a19","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.852497069Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89452023-09-22T23:27:52.943ZThe guest has requested activation with gen:68
89462023-09-22T23:27:52.946Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.852603807Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437,"task":"up_listen"}
89472023-09-22T23:27:52.949Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.852671991Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89482023-09-22T23:27:52.952Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.85271562Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89492023-09-22T23:27:52.955Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.852755387Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437,"looper":"2"}
89502023-09-22T23:27:52.958Z{"msg":"[2] b84daa34-f037-4da5-9129-454c091306d3 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.853023566Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437,"looper":"2"}
89512023-09-22T23:27:52.961Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.853093759Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437,"looper":"1"}
89522023-09-22T23:27:52.964Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.853228399Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437,"looper":"0"}
89532023-09-22T23:27:52.967Z{"msg":"[0] b84daa34-f037-4da5-9129-454c091306d3 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.853504581Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437,"looper":"0"}
89542023-09-22T23:27:52.970Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.853626705Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89552023-09-22T23:27:52.995Z{"msg":"b84daa34-f037-4da5-9129-454c091306d3 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.853683144Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89562023-09-22T23:27:52.998Z{"msg":"Set desired generation to :68","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.853707016Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89572023-09-22T23:27:53.001Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.854098563Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89582023-09-22T23:27:53.004Z{"msg":"[2] b84daa34-f037-4da5-9129-454c091306d3 (8ec6bdac-f255-453f-8a1a-9190732333db) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.854346005Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89592023-09-22T23:27:53.007Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.854465129Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89602023-09-22T23:27:53.010Z{"msg":"[2] client is_active_req TRUE, promote! session 8ec6bdac-f255-453f-8a1a-9190732333db","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.854524225Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89612023-09-22T23:27:53.013Z{"msg":"[0] b84daa34-f037-4da5-9129-454c091306d3 (8ec6bdac-f255-453f-8a1a-9190732333db) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.854547503Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89622023-09-22T23:27:53.016Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.854579926Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89632023-09-22T23:27:53.019Z{"msg":"[0] client is_active_req TRUE, promote! session 8ec6bdac-f255-453f-8a1a-9190732333db","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.854669049Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89642023-09-22T23:27:53.022Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.85506041Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89652023-09-22T23:27:53.025Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.855142163Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89662023-09-22T23:27:53.028Z{"msg":"b84daa34-f037-4da5-9129-454c091306d3 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.855189656Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89672023-09-22T23:27:53.032Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.855336333Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89682023-09-22T23:27:53.035Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.855394845Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89692023-09-22T23:27:53.038Z{"msg":"b84daa34-f037-4da5-9129-454c091306d3 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.855440851Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89702023-09-22T23:27:53.042Z{"msg":"[2] b84daa34-f037-4da5-9129-454c091306d3 (8ec6bdac-f255-453f-8a1a-9190732333db) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.857432084Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89712023-09-22T23:27:53.045Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.857488536Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89722023-09-22T23:27:53.048Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:27:52.857537567Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89732023-09-22T23:27:53.051Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.857576927Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89742023-09-22T23:27:53.054Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.857615274Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89752023-09-22T23:27:53.057Z{"msg":"b84daa34-f037-4da5-9129-454c091306d3 WaitActive New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.85765461Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89762023-09-22T23:27:53.060Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.857693764Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89772023-09-22T23:27:53.063Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.857731632Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89782023-09-22T23:27:53.066Z{"msg":"[0] b84daa34-f037-4da5-9129-454c091306d3 (8ec6bdac-f255-453f-8a1a-9190732333db) WaitActive New WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.85777258Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89792023-09-22T23:27:53.069Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.857811323Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89802023-09-22T23:27:53.072Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:27:52.857849608Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89812023-09-22T23:27:53.075Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.857887957Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89822023-09-22T23:27:53.079Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.857925953Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89832023-09-22T23:27:53.082Z{"msg":"b84daa34-f037-4da5-9129-454c091306d3 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.857964334Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89842023-09-22T23:27:53.085Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.857984278Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89852023-09-22T23:27:53.088Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:52.85800623Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89862023-09-22T23:27:53.854Z{"msg":"[1] b84daa34-f037-4da5-9129-454c091306d3 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.853877367Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437,"looper":"1"}
89872023-09-22T23:27:53.858Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.853962531Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89882023-09-22T23:27:53.861Z{"msg":"[1] b84daa34-f037-4da5-9129-454c091306d3 (8ec6bdac-f255-453f-8a1a-9190732333db) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.855149019Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89892023-09-22T23:27:53.865Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.855210959Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89902023-09-22T23:27:53.868Z{"msg":"[1] client is_active_req TRUE, promote! session 8ec6bdac-f255-453f-8a1a-9190732333db","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.855280004Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89912023-09-22T23:27:53.871Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.85592224Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89922023-09-22T23:27:53.874Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.855978886Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89932023-09-22T23:27:53.877Z{"msg":"b84daa34-f037-4da5-9129-454c091306d3 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.856021486Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89942023-09-22T23:27:53.880Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.858362039Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89952023-09-22T23:27:53.883Z{"msg":"[1] b84daa34-f037-4da5-9129-454c091306d3 (8ec6bdac-f255-453f-8a1a-9190732333db) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.858514505Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89962023-09-22T23:27:53.887Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.858573141Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89972023-09-22T23:27:53.890Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:27:53.858615331Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89982023-09-22T23:27:53.893Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.85865405Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
89992023-09-22T23:27:53.896Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.858693067Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90002023-09-22T23:27:53.900Z{"msg":"b84daa34-f037-4da5-9129-454c091306d3 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.858732423Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90012023-09-22T23:27:53.903Z{"msg":"[0]R flush_numbers[0..12]: [90, 94, 92, 92, 94, 92, 90, 98, 93, 92, 98, 96]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.858772108Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90022023-09-22T23:27:53.906Z{"msg":"[0]R generation[0..12]: [60, 66, 62, 62, 66, 62, 60, 67, 63, 62, 67, 66]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.858816645Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90032023-09-22T23:27:53.909Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.858857702Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90042023-09-22T23:27:53.912Z{"msg":"[1]R flush_numbers[0..12]: [90, 94, 92, 92, 94, 92, 90, 98, 93, 92, 98, 96]","v":0The guest has finished waiting for activation with:68
90052023-09-22T23:27:53.915Z,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
90062023-09-22T23:27:53.918Z,"time":"2023-09-22T23:27:53.858916618Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90072023-09-22T23:27:53.921Z{"msg":"[1]R generation[0..12]: [60, 66, 62, 62, 66, 62, 60, 67, 63, 62, 67, 66]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.858939625Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90082023-09-22T23:27:53.924Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.858979314Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90092023-09-22T23:27:53.927Z{"msg":"[2]R flush_numbers[0..12]: [90, 94, 92, 92, 94, 92, 90, 98, 93, 92, 98, 96]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.859026278Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90102023-09-22T23:27:53.930Z{"msg":"[2]R generation[0..12]: [60, 66, 62, 62, 66, 62, 60, 67, 63, 62, 67, 66]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.859086672Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90112023-09-22T23:27:53.934Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.85915551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90122023-09-22T23:27:53.937Z{"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.859276221Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90132023-09-22T23:27:53.942Z{"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.85933942Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90142023-09-22T23:27:53.945Z{"msg":"Next flush: 99","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.859404635Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90152023-09-22T23:27:53.949Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.859453797Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90162023-09-22T23:27:53.952Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.859512579Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90172023-09-22T23:27:53.955Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.859579426Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90182023-09-22T23:27:53.958Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.859644456Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90192023-09-22T23:27:53.961Z{"msg":"b84daa34-f037-4da5-9129-454c091306d3 is now active with session: 8ec6bdac-f255-453f-8a1a-9190732333db","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.859690258Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90202023-09-22T23:27:53.964Z{"msg":"b84daa34-f037-4da5-9129-454c091306d3 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.859750819Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90212023-09-22T23:27:53.967Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.859813174Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90222023-09-22T23:27:53.970Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.859857767Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90232023-09-22T23:27:53.974Z{"msg":"[2] received reconcile message","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
90242023-09-22T23:27:53.977Zv":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.859976379Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90252023-09-22T23:27:53.980Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.860047817Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90262023-09-22T23:27:53.983Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.860112591Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90272023-09-22T23:27:53.986Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.860174589Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90282023-09-22T23:27:53.989Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.860238978Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90292023-09-22T23:27:53.992Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.860302201Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90302023-09-22T23:27:54.018Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.860362603Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90312023-09-22T23:27:54.022Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.860422385Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90322023-09-22T23:27:54.025Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:53.860482988Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1437}
90332023-09-22T23:27:54.028ZLoading write count information from file "/tmp/test_repair_verify.data"
90342023-09-22T23:27:54.031ZRun Repair workload
90352023-09-22T23:27:54.035Z01/30 Write block 415 len 20480 data: 6 6 5 4 5
90362023-09-22T23:27:54.038Z02/30 Write block 78 len 8192 data: 7 9
90372023-09-22T23:27:54.041Z03/30 Read block 264 len 16384
90382023-09-22T23:27:54.044Z04/30 Read block 583 len 20480
90392023-09-22T23:27:54.047Z05/30 Read block 6 len 20480
90402023-09-22T23:27:54.051Z06/30 Write block 254 len 36864 data: 6 5 5 5 5 4 5 7 8
90412023-09-22T23:27:54.054Z07/30 Read block 135 len 4096
90422023-09-22T23:27:54.058Z08/30 Flush
90432023-09-22T23:27:54.061Z09/30 Write block 116 len 40960 data: 4 5 4 4 6 6 7 7 6 5
90442023-09-22T23:27:54.064Z10/30 Read block 503 len 36864
90452023-09-22T23:27:54.067Z11/30 Read block 391 len 28672
90462023-09-22T23:27:54.070Z12/30 Write block 128 len 8192 data: 5 4
90472023-09-22T23:27:54.073Z13/30 Read block 36 len 40960
90482023-09-22T23:27:54.076Z14/30 Write block 77 len 32768 data: 8 8 10 9 11 10 9 8
90492023-09-22T23:27:54.079Z15/30 Write block 463 len 40960 data: 6 5 7 5 5 5 6 4 2 2
90502023-09-22T23:27:54.082Z16/30 Read block 407 len 4096
90512023-09-22T23:27:54.085Z17/30 Write block 531 len 24576 data: 5 6 7 6 6 6
90522023-09-22T23:27:54.088Z18/30 Write block 258 len 28672 data: 6 5 6 8 9 9 9
90532023-09-22T23:27:54.091Z19/30 Write block 62 len 32768 data: 10 13 12 14 13 13 14 14
90542023-09-22T23:27:54.094Z20/30 Write block 365 len 32768 data: 3 4 5 6 5 4 4 4
90552023-09-22T23:27:54.097Z21/30 Read block 194 len 8192
90562023-09-22T23:27:54.100Z22/30 Write block 171 len 32768 data: 7 8 8 8 9 9 8 9
90572023-09-22T23:27:54.103Z23/30 Write block 335 len 8192 data: 8 7
90582023-09-22T23:27:54.107Z24/30 Write block 493 len 40960 data: 10 9 9 10 9 6 7 8 7 8
90592023-09-22T23:27:54.110Z25/30 Read block 105 len 28672
90602023-09-22T23:27:54.112Z26/30 Write block 465 len 40960 data: 8 6 6 6 7 5 3 3 3 4
90612023-09-22T23:27:54.116Z27/30 Write block 302 len 12288 data: 7 8 9
90622023-09-22T23:27:54.118Z28/30 Read block 358 len 40960
90632023-09-22T23:27:54.121Z29/30 Read block 527 len 40960
90642023-09-22T23:27:54.124Z30/30 Read block 542 len 32768
90652023-09-22T23:27:54.128Z----------------------------------------------------------------
90662023-09-22T23:27:54.131Z Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:22
90672023-09-22T23:27:54.134ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
90682023-09-22T23:27:54.137Z 9 Acked 1008 Write 10 Done Done Done false
90692023-09-22T23:27:54.140Z 10 Acked 1009 Read 9 Done Done Done false
90702023-09-22T23:27:54.144Z 11 Acked 1010 Read 7 Done Done Done false
90712023-09-22T23:27:54.147Z 12 Acked 1011 Write 2 Done Done Done false
90722023-09-22T23:27:54.150Z 13 Acked 1012 Read 10 Done Done Done false
90732023-09-22T23:27:54.153Z 14 Acked 1013 Write 8 Done Done Done false
90742023-09-22T23:27:54.156Z 15 Acked 1014 Write 10 Done Sent Done false
90752023-09-22T23:27:54.159Z 16 Acked 1015 Read 1 Done Sent Done false
90762023-09-22T23:27:54.162Z 17 Acked 1016 Write 6 Done Sent Done false
90772023-09-22T23:27:54.165Z 18 Acked 1017 Write 7 Done Sent Done false
90782023-09-22T23:27:54.168Z 19 Acked 1018 Write 8 Done Sent Done false
90792023-09-22T23:27:54.171Z 20 Acked 1019 Write 8 Done Sent Done false
90802023-09-22T23:27:54.174Z 21 Acked 1020 Read 2 Done Sent Done false
90812023-09-22T23:27:54.177Z 22 Acked 1021 Write 8 Done Sent Done false
90822023-09-22T23:27:54.180Z 23 Acked 1022 Write 2 Done Sent Done false
90832023-09-22T23:27:54.183Z 24 Acked 1023 Write 10 Done Sent Done false
90842023-09-22T23:27:54.186Z 25 Acked 1024 Read 7 Done Sent Done false
90852023-09-22T23:27:54.189Z 26 Acked 1025 Write 10 Done Sent Done false
90862023-09-22T23:27:54.192Z 27 Acked 1026 Write 3 Done Sent Done false
90872023-09-22T23:27:54.195Z 28 Acked 1027 Read 10 Done Sent Done false
90882023-09-22T23:27:54.198Z 29 Acked 1028 Read 10 Done Sent Done false
90892023-09-22T23:27:54.201Z 30 Acked 1029 Read 8 Done Sent Done false
90902023-09-22T23:27:54.205Z STATES DS:0 DS:1 DS:2 TOTAL
90912023-09-22T23:27:54.208Z New 0 0 0 0
90922023-09-22T23:27:54.211Z Sent 0 16 0 16
90932023-09-22T23:27:54.214Z Done 22 6 22 50
90942023-09-22T23:27:54.217Z Skipped 0 0 0 0
90952023-09-22T23:27:54.221Z Error 0 0 0 0
90962023-09-22T23:27:54.224ZLast Flush: 1007 1007 1007
90972023-09-22T23:27:54.227ZDownstairs last five completed: 1007 1006 1005 1004 1003 1002
90982023-09-22T23:27:54.233ZUpstairs last five completed: 30 29 28 27 26 25
90992023-09-22T23:27:54.236ZWrote out file "/tmp/test_repair_verify.data"
91002023-09-22T23:27:54.239Z
91012023-09-22T23:27:54.242ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
91022023-09-22T23:27:54.245Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:54.010157839Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1438}
91032023-09-22T23:27:54.248Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:54.010545488Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1438}
91042023-09-22T23:27:54.251Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:54.010569321Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1438}
91052023-09-22T23:27:54.254Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:54.04977119Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1438}
91062023-09-22T23:27:54.257Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:54.049818691Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1438}
91072023-09-22T23:27:54.260Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:54.049838904Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1438}
91082023-09-22T23:27:54.263Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:54.086708669Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1438}
91092023-09-22T23:27:54.266Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:54.08676345Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1438}
91102023-09-22T23:27:54.270Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:27:54.086802829Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1438}
91112023-09-22T23:27:54.272ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
91122023-09-22T23:27:54.276Z 0 000-019 60 60 60 90 90 90 F F F
91132023-09-22T23:27:54.279Z 1 020-039 66 66 66 94 94 94 F F F
91142023-09-22T23:27:54.282Z 2 040-059 62 62 62 92 92 92 F F F
91152023-09-22T23:27:54.286Z 3 060-079 68 68 68 99 99 99 T T T
91162023-09-22T23:27:54.288Z 4 080-099 66 66 66 94 94 94 T T T
91172023-09-22T23:27:54.292Z 5 100-119 62 62 62 92 92 92 T T T
91182023-09-22T23:27:54.294Z 6 120-139 60 60 60 90 90 90 T T T
91192023-09-22T23:27:54.298Z 7 140-159 67 67 67 98 98 98 F F F
91202023-09-22T23:27:54.301Z 8 160-179 63 63 63 93 93 93 T F T <---
91212023-09-22T23:27:54.304Z 9 180-199 62 62 62 92 92 92 F F F
91222023-09-22T23:27:54.307Z 10 200-219 67 67 67 98 98 98 F F F
91232023-09-22T23:27:54.310Z 11 220-239 66 66 66 96 96 96 F F F
91242023-09-22T23:27:54.313Z 12 240-259 68 68 68 99 99 99 T F T <---
91252023-09-22T23:27:54.317Z 13 260-279 68 68 68 99 99 99 T F T <---
91262023-09-22T23:27:54.320Z 14 280-299 61 61 61 91 91 91 F F F
91272023-09-22T23:27:54.323ZError: Difference in extent metadata found!
91282023-09-22T23:27:54.326Z 15 300-319 66 66 66 94 94 94 T F T <---
91292023-09-22T23:27:54.331Z 16 320-339 66 66 66 94 94 94 T F T <---
91302023-09-22T23:27:54.334Z 17 340-359 66 66 66 95 95 95 F F F
91312023-09-22T23:27:54.337Z 18 360-379 63 63 63 93 93 93 T F T <---
91322023-09-22T23:27:54.340Z 19 380-399 63 63 63 93 93 93 F F F
91332023-09-22T23:27:54.343Z 20 400-419 68 68 68 99 99 99 F F F
91342023-09-22T23:27:54.346Z 21 420-439 63 63 63 93 93 93 F F F
91352023-09-22T23:27:54.349Z 22 440-459 63 63 63 93 93 93 F F F
91362023-09-22T23:27:54.353Z 23 460-479 66 66 66 95 95 95 T F T <---
91372023-09-22T23:27:54.356Z 24 480-499 66 66 66 95 95 95 T F T <---
91382023-09-22T23:27:54.359Z 25 500-519 61 61 61 91 91 91 T F T <---
91392023-09-22T23:27:54.362Z 26 520-539 63 63 63 93 93 93 T F T <---
91402023-09-22T23:27:54.365Z 27 540-559 58 58 58 86 86 86 F F F
91412023-09-22T23:27:54.368Z 28 560-579 66 66 66 94 94 94 F F F
91422023-09-22T23:27:54.371Z 29 580-599 59 59 59 89 89 89 F F F
91432023-09-22T23:27:54.374ZMax gen: 68, Max flush: 99
91442023-09-22T23:27:54.377ZA Difference in extent metadata is expected here
91452023-09-22T23:27:54.380ZOn loop 34
91462023-09-22T23:27:54.383Z
91472023-09-22T23:27:54.386ZVerifying data now
91482023-09-22T23:28:00.645ZNo change after verify
91492023-09-22T23:28:00.648ZLoop: 34 Downstairs dump after verify (and repair):
91502023-09-22T23:28:00.664Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.66209523Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1443}
91512023-09-22T23:28:00.667Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.662502778Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1443}
91522023-09-22T23:28:00.670Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.662528756Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1443}
91532023-09-22T23:28:00.682Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.681450345Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1443}
91542023-09-22T23:28:00.686Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.681505035Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1443}
91552023-09-22T23:28:00.689Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.681525568Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1443}
91562023-09-22T23:28:00.694Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.692711071Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1443}
91572023-09-22T23:28:00.697Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.692764784Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1443}
91582023-09-22T23:28:00.701Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.692786275Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1443}
91592023-09-22T23:28:00.704ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
91602023-09-22T23:28:00.707Z 0 000-019 60 60 60 90 90 90 F F F
91612023-09-22T23:28:00.711Z 1 020-039 66 66 66 94 94 94 F F F
91622023-09-22T23:28:00.714Z 2 040-059 62 62 62 92 92 92 F F F
91632023-09-22T23:28:00.717Z 3 060-079 69 69 69 100 100 100 F F F
91642023-09-22T23:28:00.721Z 4 080-099 69 69 69 100 100 100 F F F
91652023-09-22T23:28:00.724Z 5 100-119 69 69 69 100 100 100 F F F
91662023-09-22T23:28:00.727Z 6 120-139 69 69 69 100 100 100 F F F
91672023-09-22T23:28:00.730Z 7 140-159 67 67 67 98 98 98 F F F
91682023-09-22T23:28:00.733Z 8 160-179 69 69 69 100 100 100 F F F
91692023-09-22T23:28:00.737Z 9 180-199 62 62 62 92 92 92 F F F
91702023-09-22T23:28:00.740Z 10 200-219 67 67 67 98 98 98 F F F
91712023-09-22T23:28:00.743Z 11 220-239 66 66 66 96 96 96 F F F
91722023-09-22T23:28:00.746Z 12 240-259 69 69 69 100 100 100 F F F
91732023-09-22T23:28:00.749Z 13 260-279 69 69 69 100 100 100 F F F
91742023-09-22T23:28:00.752Z 14 280-299 61 61 61 91 91 91 F F F
91752023-09-22T23:28:00.755Z 15 300-319 69 69 69 100 100 100 F F F
91762023-09-22T23:28:00.758Z 16 320-339 69 69 69 100 100 100 F F F
91772023-09-22T23:28:00.763Z 17 340-359 66 66 66 95 95 95 F F F
91782023-09-22T23:28:00.767Z 18 360-379 69 69 69 100 100 100 F F F
91792023-09-22T23:28:00.770Z 19 380-399 63 63 63 93 93 93 F F F
91802023-09-22T23:28:00.774Z 20 400-419 68 68 68 99 99 99 F F F
91812023-09-22T23:28:00.776Z 21 420-439 63 63 63 93 93 93 F F F
91822023-09-22T23:28:00.780Z 22 440-459 63 63 63 93 93 93 F F F
91832023-09-22T23:28:00.783Z 23 460-479 69 69 69 100 100 100 F F F
91842023-09-22T23:28:00.786Z 24 480-499 69 69 69 100 100 100 F F F
91852023-09-22T23:28:00.789Z 25 500-519 69 69 69 100 100 100 F F F
91862023-09-22T23:28:00.793Z 26 520-539 69 69 69 100 100 100 F F F
91872023-09-22T23:28:00.797Z 27 540-559 58 58 58 86 86 86 F F F
91882023-09-22T23:28:00.800Z 28 560-579 66 66 66 94 94 94 F F F
91892023-09-22T23:28:00.803Z 29 580-599 59 59 59 89 89 89 F F F
91902023-09-22T23:28:00.806ZMax gen: 69, Max flush: 100
91912023-09-22T23:28:00.809Z
91922023-09-22T23:28:00.813ZBegin loop 35
91932023-09-22T23:28:00.816ZDownstairs to restart: 2
91942023-09-22T23:28:00.819Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.741953169Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
91952023-09-22T23:28:00.823Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.742335237Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
91962023-09-22T23:28:00.826Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.742395155Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
91972023-09-22T23:28:00.829Z{"msg":"Crucible stats registered with UUID: 84034dc1-3203-440c-9806-e7a792a32b4d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.742420528Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
91982023-09-22T23:28:00.832Z{"msg":"Crucible 84034dc1-3203-440c-9806-e7a792a32b4d has session id: df581504-4da2-4c9a-a5b6-0bcecc384c94","v":0,"name":"crucible","level":30Crucible runtime is spawned
91992023-09-22T23:28:00.835Z,"time":"2023-09-22T23:28:00.742551622Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92002023-09-22T23:28:00.838Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.74271358Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445,"task":"up_listen"}
92012023-09-22T23:28:00.842Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30The guest has requested activation with gen:70
92022023-09-22T23:28:00.845Z,"time":"2023-09-22T23:28:00.742804332Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92032023-09-22T23:28:00.849Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.74292621Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92042023-09-22T23:28:00.852Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.742973434Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445,"looper":"0"}
92052023-09-22T23:28:00.856Z{"msg":"[0] 84034dc1-3203-440c-9806-e7a792a32b4d looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.743179544Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445,"looper":"0"}
92062023-09-22T23:28:00.859Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.743257702Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445,"looper":"2"}
92072023-09-22T23:28:00.862Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.743430452Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445,"looper":"1"}
92082023-09-22T23:28:00.865Z{"msg":"[1] 84034dc1-3203-440c-9806-e7a792a32b4d looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.743650067Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445,"looper":"1"}
92092023-09-22T23:28:00.868Z{"msg":"84034dc1-3203-440c-9806-e7a792a32b4d active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.743894168Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92102023-09-22T23:28:00.871Z{"msg":"Set desired generation to :70","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.743954027Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92112023-09-22T23:28:00.876Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.744027262Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92122023-09-22T23:28:00.879Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.744361323Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92132023-09-22T23:28:00.882Z{"msg":"[0] 84034dc1-3203-440c-9806-e7a792a32b4d (f21cbe49-327c-4a73-8f76-07c57fc594f0) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.744727716Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92142023-09-22T23:28:00.886Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.74479622Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92152023-09-22T23:28:00.889Z{"msg":"[0] client is_active_req TRUE, promote! session f21cbe49-327c-4a73-8f76-07c57fc594f0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.744856864Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92162023-09-22T23:28:00.893Z{"msg":"[1] 84034dc1-3203-440c-9806-e7a792a32b4d (f21cbe49-327c-4a73-8f76-07c57fc594f0) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.744932219Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92172023-09-22T23:28:00.896Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.744987294Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92182023-09-22T23:28:00.899Z{"msg":"[1] client is_active_req TRUE, promote! session f21cbe49-327c-4a73-8f76-07c57fc594f0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.745044308Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92192023-09-22T23:28:00.902Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.745291449Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92202023-09-22T23:28:00.906Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.745404246Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92212023-09-22T23:28:00.909Z{"msg":"84034dc1-3203-440c-9806-e7a792a32b4d WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.745486285Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92222023-09-22T23:28:00.913Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.745582227Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92232023-09-22T23:28:00.916Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.745664631Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92242023-09-22T23:28:00.919Z{"msg":"84034dc1-3203-440c-9806-e7a792a32b4d WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.745738456Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92252023-09-22T23:28:00.922Z{"msg":"[0] 84034dc1-3203-440c-9806-e7a792a32b4d (f21cbe49-327c-4a73-8f76-07c57fc594f0) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.747795914Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92262023-09-22T23:28:00.925Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.747935412Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92272023-09-22T23:28:00.931Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:28:00.748003461Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92282023-09-22T23:28:00.935Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.748059853Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92292023-09-22T23:28:00.937Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.748114891Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92302023-09-22T23:28:00.940Z{"msg":"84034dc1-3203-440c-9806-e7a792a32b4d WaitQuorum WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.748170299Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92312023-09-22T23:28:00.943Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.748224914Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92322023-09-22T23:28:00.946Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.748278888Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92332023-09-22T23:28:00.950Z{"msg":"[1] 84034dc1-3203-440c-9806-e7a792a32b4d (f21cbe49-327c-4a73-8f76-07c57fc594f0) WaitQuorum WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.748337362Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92342023-09-22T23:28:00.953Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.748391772Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92352023-09-22T23:28:00.958Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:28:00.748445772Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92362023-09-22T23:28:00.961Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.74850013Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92372023-09-22T23:28:00.967Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.748555373Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92382023-09-22T23:28:00.970Z{"msg":"84034dc1-3203-440c-9806-e7a792a32b4d WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.748610438Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92392023-09-22T23:28:00.973Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.748640338Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92402023-09-22T23:28:00.976Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:00.748659256Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92412023-09-22T23:28:01.746Z{"msg":"[2] 84034dc1-3203-440c-9806-e7a792a32b4d looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.745301887Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445,"looper":"2"}
92422023-09-22T23:28:01.750Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.745409806Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92432023-09-22T23:28:01.754Z{"msg":"[2] 84034dc1-3203-440c-9806-e7a792a32b4d (f21cbe49-327c-4a73-8f76-07c57fc594f0) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.746359028Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92442023-09-22T23:28:01.758Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.746426967Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92452023-09-22T23:28:01.761Z{"msg":"[2] client is_active_req TRUE, promote! session f21cbe49-327c-4a73-8f76-07c57fc594f0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.746486687Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92462023-09-22T23:28:01.765Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.747016409Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92472023-09-22T23:28:01.768Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.747080711Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92482023-09-22T23:28:01.772Z{"msg":"84034dc1-3203-440c-9806-e7a792a32b4d WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.747124226Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92492023-09-22T23:28:01.776Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.749263515Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92502023-09-22T23:28:01.779Z{"msg":"[2] 84034dc1-3203-440c-9806-e7a792a32b4d (f21cbe49-327c-4a73-8f76-07c57fc594f0) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.749416402Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92512023-09-22T23:28:01.783Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.749550879Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92522023-09-22T23:28:01.786Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:28:01.749618716Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92532023-09-22T23:28:01.790Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.749678995Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92542023-09-22T23:28:01.793Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.749722569Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92552023-09-22T23:28:01.800Z{"msg":"84034dc1-3203-440c-9806-e7a792a32b4d WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.749762692Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92562023-09-22T23:28:01.803Z{"msg":"[0]R flush_numbers[0..12]: [90, 94, 92, 100, 100, 100, 100, 98, 100, 92, 98, 96]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.749803667Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92572023-09-22T23:28:01.807Z{"The guest has finished waiting for activation with:70
92582023-09-22T23:28:01.810ZWait for a query_work_queue command to finish before sending IO
92592023-09-22T23:28:01.814Zmsg":"[0]R generation[0..12]: [60, 66, 62, 69, 69, 69, 69, 67, 69, 62, 67, 66]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.750015271Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92602023-09-22T23:28:01.817Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.750096483Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92612023-09-22T23:28:01.820Z{"msg":"[1]R flush_numbers[0..12]: [90, 94, 92, 100, 100, 100, 100, 98, 100, 92, 98, 96]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.750148847Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92622023-09-22T23:28:01.823Z{"msg":"[1]R generation[0..12]: [60, 66, 62, 69, 69, 69, 69, 67, 69, 62, 67, 66]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.75019013Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92632023-09-22T23:28:01.826Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.750230885Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92642023-09-22T23:28:01.830Z{"msg":"[2]R flush_numbers[0..12]: [90, 94, 92, 100, 100, 100, 100, 98, 100, 92, 98, 96]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.750276585Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92652023-09-22T23:28:01.833Z{"msg":"[2]R generation[0..12]: [60, 66, 62, 69, 69, 69, 69, 67, 69, 62, 67, 66]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.750316032Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92662023-09-22T23:28:01.837Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.750361594Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92672023-09-22T23:28:01.840Z{"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.750399523Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92682023-09-22T23:28:01.844Z{"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.75043899Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92692023-09-22T23:28:01.848Z{"msg":"Next flush: 101","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.750477391Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92702023-09-22T23:28:01.851Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.750498154Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92712023-09-22T23:28:01.854Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.750539087Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92722023-09-22T23:28:01.858Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.75057704Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92732023-09-22T23:28:01.861Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.75061548Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92742023-09-22T23:28:01.866Z{"msg":"84034dc1-3203-440c-9806-e7a792a32b4d is now active with session: f21cbe49-327c-4a73-8f76-07c57fc594f0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.750657337Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92752023-09-22T23:28:01.870Z{"msg":"84034dc1-3203-440c-9806-e7a792a32b4d Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.750707288Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92762023-09-22T23:28:01.874Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.750745597Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92772023-09-22T23:28:01.878Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.750783316Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92782023-09-22T23:28:01.881ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
92792023-09-22T23:28:01.885Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.750826178Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92802023-09-22T23:28:01.889Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.7508638Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92812023-09-22T23:28:01.893Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.750912184Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92822023-09-22T23:28:01.897Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.750951622Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92832023-09-22T23:28:01.900Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.750989075Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92842023-09-22T23:28:01.904Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.751026462Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92852023-09-22T23:28:01.908Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.751063942Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92862023-09-22T23:28:01.911Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.75110132Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92872023-09-22T23:28:01.914Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.751138635Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1445}
92882023-09-22T23:28:01.917ZLoading write count information from file "/tmp/test_repair_verify.data"
92892023-09-22T23:28:01.920ZRun Repair workload
92902023-09-22T23:28:01.924Z01/30 Write block 497 len 4096 data: 10
92912023-09-22T23:28:01.927Z02/30 Read block 118 len 8192
92922023-09-22T23:28:01.931Z03/30 Write block 417 len 28672 data: 6 5 6 6 6 5 4
92932023-09-22T23:28:01.936Z04/30 Read block 154 len 8192
92942023-09-22T23:28:01.939Z05/30 Read block 583 len 24576
92952023-09-22T23:28:01.942Z06/30 Write block 381 len 8192 data: 7 6
92962023-09-22T23:28:01.946Z07/30 Read block 29 len 28672
92972023-09-22T23:28:01.949Z08/30 Write block 429 len 8192 data: 7 7
92982023-09-22T23:28:01.953Z09/30 Flush
92992023-09-22T23:28:01.957Z10/30 Write block 148 len 20480 data: 4 5 5 6 5
93002023-09-22T23:28:01.960Z11/30 Read block 398 len 16384
93012023-09-22T23:28:01.964Z12/30 Read block 322 len 40960
93022023-09-22T23:28:01.968Z13/30 Read block 521 len 16384
93032023-09-22T23:28:01.971Z14/30 Read block 340 len 28672
93042023-09-22T23:28:01.975Z15/30 Flush
93052023-09-22T23:28:01.978Z16/30 Write block 103 len 32768 data: 6 8 10 10 9 8 8 9
93062023-09-22T23:28:01.981Z17/30 Read block 63 len 20480
93072023-09-22T23:28:01.984Z18/30 Flush
93082023-09-22T23:28:01.988Z19/30 Write block 371 len 40960 data: 5 5 5 6 6 6 8 8 7 7
93092023-09-22T23:28:01.991Z20/30 Write block 459 len 40960 data: 7 6 7 6 7 6 9 7 7 7
93102023-09-22T23:28:01.995Z21/30 Read block 590 len 28672
93112023-09-22T23:28:01.999Z22/30 Read block 308 len 28672
93122023-09-22T23:28:02.004Z23/30 Read block 512 len 8192
93132023-09-22T23:28:02.007Z24/30 Read block 504 len 36864
93142023-09-22T23:28:02.011Z25/30 Write block 558 len 36864 data: 5 5 5 5 3 3 6 5 6
93152023-09-22T23:28:02.015Z26/30 Write block 549 len 16384 data: 10 9 10 9
93162023-09-22T23:28:02.018Z27/30 Read block 1 len 4096
93172023-09-22T23:28:02.021Z28/30 Read block 508 len 4096
93182023-09-22T23:28:02.024Z29/30 Write block 197 len 40960 data: 7 5 4 5 4 6 6 5 9 10
93192023-09-22T23:28:02.027Z30/30 Read block 40 len 12288
93202023-09-22T23:28:02.031Z----------------------------------------------------------------
93212023-09-22T23:28:02.035Z Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:21
93222023-09-22T23:28:02.038ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
93232023-09-22T23:28:02.042Z 10 Acked 1009 Write 5 Done Done Sent false
93242023-09-22T23:28:02.046Z 11 Acked 1010 Read 4 Done Done Sent false
93252023-09-22T23:28:02.049Z 12 Acked 1011 Read 10 Done Done Sent false
93262023-09-22T23:28:02.052Z 13 Acked 1012 Read 4 Done Done Sent false
93272023-09-22T23:28:02.057Z 14 Acked 1013 Read 7 Done Done Sent false
93282023-09-22T23:28:02.060Z 15 Acked 1014 Flush 0 Done Done Sent false
93292023-09-22T23:28:02.065Z 16 Acked 1015 Write 8 Done Done Sent false
93302023-09-22T23:28:02.068Z 17 Acked 1016 Read 5 Done Done Sent false
93312023-09-22T23:28:02.072Z 18 Acked 1017 Flush 0 Done Done Sent false
93322023-09-22T23:28:02.076Z 19 Acked 1018 Write 10 Done Done Sent false
93332023-09-22T23:28:02.079Z 20 Acked 1019 Write 10 Done Done Sent false
93342023-09-22T23:28:02.082Z 21 Acked 1020 Read 7 Done Done Sent false
93352023-09-22T23:28:02.085Z 22 Acked 1021 Read 7 Done Done Sent false
93362023-09-22T23:28:02.089Z 23 Acked 1022 Read 2 Done Done Sent false
93372023-09-22T23:28:02.093Z 24 Acked 1023 Read 9 Done Done Sent false
93382023-09-22T23:28:02.096Z 25 Acked 1024 Write 9 Done Done Sent false
93392023-09-22T23:28:02.099Z 26 Acked 1025 Write 4 Done Done Sent false
93402023-09-22T23:28:02.103Z 27 Acked 1026 Read 1 Done Done Sent false
93412023-09-22T23:28:02.106Z 28 Acked 1027 Read 1 Done Done Sent false
93422023-09-22T23:28:02.109Z 29 Acked 1028 Write 10 Done Done Sent false
93432023-09-22T23:28:02.112Z 30 Acked 1029 Read 3 Sent Done Sent false
93442023-09-22T23:28:02.116Z STATES DS:0 DS:1 DS:2 TOTAL
93452023-09-22T23:28:02.120Z New 0 0 0 0
93462023-09-22T23:28:02.124Z Sent 1 0 21 22
93472023-09-22T23:28:02.127Z Done 20 21 0 41
93482023-09-22T23:28:02.131Z Skipped 0 0 0 0
93492023-09-22T23:28:02.135Z Error 0 0 0 0
93502023-09-22T23:28:02.138ZLast Flush: 1017 1017 1008
93512023-09-22T23:28:02.141ZDownstairs last five completed: 1008 1007 1006 1005 1004 1003
93522023-09-22T23:28:02.145ZUpstairs last five completed: 30 29 28 27 26 25
93532023-09-22T23:28:02.149ZWrote out file "/tmp/test_repair_verify.data"
93542023-09-22T23:28:02.152Z
93552023-09-22T23:28:02.156ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
93562023-09-22T23:28:02.159Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.894996038Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1446}
93572023-09-22T23:28:02.163Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.89525909Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1446}
93582023-09-22T23:28:02.166Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.895282755Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1446}
93592023-09-22T23:28:02.191Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.927553227Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1446}
93602023-09-22T23:28:02.195Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.927588188Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1446}
93612023-09-22T23:28:02.198Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.927608112Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1446}
93622023-09-22T23:28:02.202Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.950617391Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1446}
93632023-09-22T23:28:02.206Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.950673624Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1446}
93642023-09-22T23:28:02.210Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:01.950694934Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1446}
93652023-09-22T23:28:02.213ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
93662023-09-22T23:28:02.216Z 0 000-019 60 60 60 90 90 90 F F F
93672023-09-22T23:28:02.219Z 1 020-039 66 66 66 94 94 94 F F F
93682023-09-22T23:28:02.222Z 2 040-059 62 62 62 92 92 92 F F F
93692023-09-22T23:28:02.227Z 3 060-079 69 69 69 100 100 100 F F F
93702023-09-22T23:28:02.230ZError: Difference in extent metadata found!
93712023-09-22T23:28:02.234Z 4 080-099 69 69 69 100 100 100 F F F
93722023-09-22T23:28:02.237Z 5 100-119 70 70 69 103 103 100 F F F <---
93732023-09-22T23:28:02.240Z 6 120-139 69 69 69 100 100 100 F F F
93742023-09-22T23:28:02.243Z 7 140-159 70 70 67 102 102 98 F F F <---
93752023-09-22T23:28:02.247Z 8 160-179 69 69 69 100 100 100 F F F
93762023-09-22T23:28:02.251Z 9 180-199 62 62 62 92 92 92 T T F <---
93772023-09-22T23:28:02.660Z 10 200-219 67 67 67 98 98 98 T T F <---
93782023-09-22T23:28:02.664Z 11 220-239 66 66 66 96 96 96 F F F
93792023-09-22T23:28:02.667Z 12 240-259 69 69 69 100 100 100 F F F
93802023-09-22T23:28:02.670Z 13 260-279 69 69 69 100 100 100 F F F
93812023-09-22T23:28:02.674Z 14 280-299 61 61 61 91 91 91 F F F
93822023-09-22T23:28:02.678Z 15 300-319 69 69 69 100 100 100 F F F
93832023-09-22T23:28:02.681Z 16 320-339 69 69 69 100 100 100 F F F
93842023-09-22T23:28:02.684Z 17 340-359 66 66 66 95 95 95 F F F
93852023-09-22T23:28:02.688Z 18 360-379 69 69 69 100 100 100 T T F <---
93862023-09-22T23:28:02.692Z 19 380-399 70 70 70 101 101 101 T T F <---
93872023-09-22T23:28:02.697Z 20 400-419 70 70 70 101 101 101 F F F
93882023-09-22T23:28:02.700Z 21 420-439 70 70 70 101 101 101 F F F
93892023-09-22T23:28:02.703Z 22 440-459 63 63 63 93 93 93 T T F <---
93902023-09-22T23:28:02.706Z 23 460-479 69 69 69 100 100 100 T T F <---
93912023-09-22T23:28:02.710Z 24 480-499 70 70 70 101 101 101 F F F
93922023-09-22T23:28:02.714Z 25 500-519 69 69 69 100 100 100 F F F
93932023-09-22T23:28:02.717Z 26 520-539 69 69 69 100 100 100 F F F
93942023-09-22T23:28:02.720Z 27 540-559 58 58 58 86 86 86 T T F <---
93952023-09-22T23:28:02.724Z 28 560-579 66 66 66 94 94 94 T T F <---
93962023-09-22T23:28:02.727Z 29 580-599 59 59 59 89 89 89 F F F
93972023-09-22T23:28:02.732ZMax gen: 70, Max flush: 103
93982023-09-22T23:28:02.737ZA Difference in extent metadata is expected here
93992023-09-22T23:28:02.740ZOn loop 35
94002023-09-22T23:28:02.744Z
94012023-09-22T23:28:02.747ZVerifying data now
94022023-09-22T23:28:08.153ZNo change after verify
94032023-09-22T23:28:08.156ZLoop: 35 Downstairs dump after verify (and repair):
94042023-09-22T23:28:08.170Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.169473008Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1451}
94052023-09-22T23:28:08.173Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.169849362Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1451}
94062023-09-22T23:28:08.176Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.169873816Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1451}
94072023-09-22T23:28:08.190Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.18883538Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1451}
94082023-09-22T23:28:08.193Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.188892619Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1451}
94092023-09-22T23:28:08.196Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.188914099Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1451}
94102023-09-22T23:28:08.201Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.200067395Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1451}
94112023-09-22T23:28:08.205Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.200119334Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1451}
94122023-09-22T23:28:08.208Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.200139972Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1451}
94132023-09-22T23:28:08.212ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
94142023-09-22T23:28:08.215Z 0 000-019 60 60 60 90 90 90 F F F
94152023-09-22T23:28:08.218Z 1 020-039 66 66 66 94 94 94 F F F
94162023-09-22T23:28:08.221Z 2 040-059 62 62 62 92 92 92 F F F
94172023-09-22T23:28:08.224Z 3 060-079 69 69 69 100 100 100 F F F
94182023-09-22T23:28:08.227Z 4 080-099 69 69 69 100 100 100 F F F
94192023-09-22T23:28:08.230Z 5 100-119 70 70 70 103 103 103 F F F
94202023-09-22T23:28:08.233Z 6 120-139 69 69 69 100 100 100 F F F
94212023-09-22T23:28:08.237Z 7 140-159 70 70 70 102 102 102 F F F
94222023-09-22T23:28:08.239Z 8 160-179 69 69 69 100 100 100 F F F
94232023-09-22T23:28:08.243Z 9 180-199 71 71 71 104 104 104 F F F
94242023-09-22T23:28:08.246Z 10 200-219 71 71 71 104 104 104 F F F
94252023-09-22T23:28:08.249Z 11 220-239 66 66 66 96 96 96 F F F
94262023-09-22T23:28:08.252Z 12 240-259 69 69 69 100 100 100 F F F
94272023-09-22T23:28:08.255Z 13 260-279 69 69 69 100 100 100 F F F
94282023-09-22T23:28:08.258Z 14 280-299 61 61 61 91 91 91 F F F
94292023-09-22T23:28:08.261Z 15 300-319 69 69 69 100 100 100 F F F
94302023-09-22T23:28:08.264Z 16 320-339 69 69 69 100 100 100 F F F
94312023-09-22T23:28:08.269Z 17 340-359 66 66 66 95 95 95 F F F
94322023-09-22T23:28:08.271Z 18 360-379 71 71 71 104 104 104 F F F
94332023-09-22T23:28:08.274Z 19 380-399 71 71 71 104 104 104 F F F
94342023-09-22T23:28:08.277Z 20 400-419 70 70 70 101 101 101 F F F
94352023-09-22T23:28:08.280Z 21 420-439 70 70 70 101 101 101 F F F
94362023-09-22T23:28:08.283Z 22 440-459 71 71 71 104 104 104 F F F
94372023-09-22T23:28:08.287Z 23 460-479 71 71 71 104 104 104 F F F
94382023-09-22T23:28:08.290Z 24 480-499 70 70 70 101 101 101 F F F
94392023-09-22T23:28:08.293Z 25 500-519 69 69 69 100 100 100 F F F
94402023-09-22T23:28:08.296Z 26 520-539 69 69 69 100 100 100 F F F
94412023-09-22T23:28:08.321Z 27 540-559 71 71 71 104 104 104 F F F
94422023-09-22T23:28:08.324Z 28 560-579 71 71 71 104 104 104 F F F
94432023-09-22T23:28:08.328Z 29 580-599 59 59 59 89 89 89 F F F
94442023-09-22T23:28:08.331ZMax gen: 71, Max flush: 104
94452023-09-22T23:28:08.334Z
94462023-09-22T23:28:08.337ZBegin loop 36
94472023-09-22T23:28:08.340ZDownstairs to restart: 0
94482023-09-22T23:28:08.343Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.249002324Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94492023-09-22T23:28:08.345Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.249369861Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94502023-09-22T23:28:08.348Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.249421669Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94512023-09-22T23:28:08.351Z{"msg":"Crucible stats registered with UUID: d8dd4776-da89-4da3-b671-448d598fa220","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.249527227Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94522023-09-22T23:28:08.354Z{"msg":"Crucible d8dd4776-da89-4da3-b671-448d598fa220 has session id: af3a4b0a-f638-4d75-9a13-d928712d1948","v":0,"name":"crucible","level":30,"time":"Crucible runtime is spawned
94532023-09-22T23:28:08.357Z2023-09-22T23:28:08.249563721Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94542023-09-22T23:28:08.360ZThe guest has requested activation with gen:72
94552023-09-22T23:28:08.363Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.249839705Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453,"task":"up_listen"}
94562023-09-22T23:28:08.366Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.249902716Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94572023-09-22T23:28:08.369Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.249926836Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94582023-09-22T23:28:08.372Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.249964347Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453,"looper":"2"}
94592023-09-22T23:28:08.375Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.250242162Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453,"looper":"0"}
94602023-09-22T23:28:08.378Z{"msg":"[2] d8dd4776-da89-4da3-b671-448d598fa220 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.250321996Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453,"looper":"2"}
94612023-09-22T23:28:08.381Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.250372584Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453,"looper":"1"}
94622023-09-22T23:28:08.384Z{"msg":"[1] d8dd4776-da89-4da3-b671-448d598fa220 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.25060421Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453,"looper":"1"}
94632023-09-22T23:28:08.387Z{"msg":"d8dd4776-da89-4da3-b671-448d598fa220 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.250910167Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94642023-09-22T23:28:08.390Z{"msg":"Set desired generation to :72","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.250971956Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94652023-09-22T23:28:08.393Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.251016817Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94662023-09-22T23:28:08.396Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.251375483Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94672023-09-22T23:28:08.399Z{"msg":"[2] d8dd4776-da89-4da3-b671-448d598fa220 (43259d91-723f-441c-bf2d-b971c3dc784a) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.251694822Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94682023-09-22T23:28:08.402Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.251790817Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94692023-09-22T23:28:08.405Z{"msg":"[2] client is_active_req TRUE, promote! session 43259d91-723f-441c-bf2d-b971c3dc784a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.251814751Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94702023-09-22T23:28:08.408Z{"msg":"[1] d8dd4776-da89-4da3-b671-448d598fa220 (43259d91-723f-441c-bf2d-b971c3dc784a) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.251855239Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94712023-09-22T23:28:08.411Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.251905176Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94722023-09-22T23:28:08.414Z{"msg":"[1] client is_active_req TRUE, promote! session 43259d91-723f-441c-bf2d-b971c3dc784a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.25197218Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94732023-09-22T23:28:08.417Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.252341486Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94742023-09-22T23:28:08.420Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.252413098Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94752023-09-22T23:28:08.423Z{"msg":"d8dd4776-da89-4da3-b671-448d598fa220 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.252455977Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94762023-09-22T23:28:08.427Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.252576137Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94772023-09-22T23:28:08.430Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.2526309Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94782023-09-22T23:28:08.433Z{"msg":"d8dd4776-da89-4da3-b671-448d598fa220 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.252682922Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94792023-09-22T23:28:08.436Z{"msg":"[2] d8dd4776-da89-4da3-b671-448d598fa220 (43259d91-723f-441c-bf2d-b971c3dc784a) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.254525716Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94802023-09-22T23:28:08.438Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.25460941Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94812023-09-22T23:28:08.441Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:28:08.254707809Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94822023-09-22T23:28:08.444Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.254749488Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94832023-09-22T23:28:08.447Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.254788674Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94842023-09-22T23:28:08.450Z{"msg":"d8dd4776-da89-4da3-b671-448d598fa220 New WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.254828336Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94852023-09-22T23:28:08.453Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.254921438Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94862023-09-22T23:28:08.456Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.254964854Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94872023-09-22T23:28:08.459Z{"msg":"[1] d8dd4776-da89-4da3-b671-448d598fa220 (43259d91-723f-441c-bf2d-b971c3dc784a) New WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.255013332Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94882023-09-22T23:28:08.462Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.25506953Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94892023-09-22T23:28:08.465Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:28:08.25513057Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94902023-09-22T23:28:08.469Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.255160513Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94912023-09-22T23:28:08.472Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.255213016Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94922023-09-22T23:28:08.474Z{"msg":"d8dd4776-da89-4da3-b671-448d598fa220 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.255256665Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94932023-09-22T23:28:08.479Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.255295811Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94942023-09-22T23:28:08.482Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:08.255315372Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94952023-09-22T23:28:09.253Z{"msg":"[0] d8dd4776-da89-4da3-b671-448d598fa220 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.252061302Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453,"looper":"0"}
94962023-09-22T23:28:09.256Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.252150168Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94972023-09-22T23:28:09.258Z{"msg":"[0] d8dd4776-da89-4da3-b671-448d598fa220 (43259d91-723f-441c-bf2d-b971c3dc784a) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.253327485Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94982023-09-22T23:28:09.261Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.25338592Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
94992023-09-22T23:28:09.265Z{"msg":"[0] client is_active_req TRUE, promote! session 43259d91-723f-441c-bf2d-b971c3dc784a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.253411558Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95002023-09-22T23:28:09.269Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.254046392Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95012023-09-22T23:28:09.271Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.254109598Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95022023-09-22T23:28:09.274Z{"msg":"d8dd4776-da89-4da3-b671-448d598fa220 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.254151549Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95032023-09-22T23:28:09.277Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.255324565Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95042023-09-22T23:28:09.280Z{"msg":"[0] d8dd4776-da89-4da3-b671-448d598fa220 (43259d91-723f-441c-bf2d-b971c3dc784a) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.256592011Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95052023-09-22T23:28:09.283Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.256632625Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95062023-09-22T23:28:09.286Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:28:09.256687311Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95072023-09-22T23:28:09.289Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.25671649Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95082023-09-22T23:28:09.292Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.256759858Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95092023-09-22T23:28:09.295Z{"msg":"d8dd4776-da89-4da3-b671-448d598fa220 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.256842304Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95102023-09-22T23:28:09.298Z{"msg":"[0]R flush_numbers[0..12]: [90, 94, 92, 100, 100, 103, 100, 102, 100, 104, 104, 96]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.256883552Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95112023-09-22T23:28:09.301Z{"msg":"[0]R generation[0..12]: [60, 66, 62, 69, 69, 70, 69, 70, 69, 71, 71, 66]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.256923393Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95122023-09-22T23:28:09.304Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.256964076Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95132023-09-22T23:28:09.306Z{"msg":"[1]R flush_numbers[0..12]: [90, 94, 92, 100, 100, 103, 100, 102, 100, 104, 104, 96]"The guest has finished waiting for activation with:72
95142023-09-22T23:28:09.309Z,"v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
95152023-09-22T23:28:09.312Z,"time":"2023-09-22T23:28:09.25701317Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95162023-09-22T23:28:09.315Z{"msg":"[1]R generation[0..12]: [60, 66, 62, 69, 69, 70, 69, 70, 69, 71, 71, 66]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.257059625Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95172023-09-22T23:28:09.317Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.257128856Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95182023-09-22T23:28:09.342Z{"msg":"[2]R flush_numbers[0..12]: [90, 94, 92, 100, 100, 103, 100, 102, 100, 104, 104, 96]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.257177736Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95192023-09-22T23:28:09.348Z{"msg":"[2]R generation[0..12]: [60, 66, 62, 69, 69, 70, 69, 70, 69, 71, 71, 66]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.25725291Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95202023-09-22T23:28:09.351Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.257319396Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95212023-09-22T23:28:09.353Z{"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.257363584Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95222023-09-22T23:28:09.356Z{"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.257403331Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95232023-09-22T23:28:09.359Z{"msg":"Next flush: 105","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.257444916Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95242023-09-22T23:28:09.362Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.257509281Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95252023-09-22T23:28:09.366Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.257554152Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95262023-09-22T23:28:09.369Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.257593553Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95272023-09-22T23:28:09.371Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.257613358Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95282023-09-22T23:28:09.374Z{"msg":"d8dd4776-da89-4da3-b671-448d598fa220 is now active with session: 43259d91-723f-441c-bf2d-b971c3dc784a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.257647471Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95292023-09-22T23:28:09.377Z{"msg":"d8dd4776-da89-4da3-b671-448d598fa220 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.257703527Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95302023-09-22T23:28:09.380Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.257726756Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95312023-09-22T23:28:09.383Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.257767855Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95322023-09-22T23:28:09.386Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
95332023-09-22T23:28:09.389Z[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.25783524Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95342023-09-22T23:28:09.392Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.257865617Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95352023-09-22T23:28:09.395Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.257905993Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95362023-09-22T23:28:09.398Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.257944624Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95372023-09-22T23:28:09.401Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.257982006Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95382023-09-22T23:28:09.403Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.258019577Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95392023-09-22T23:28:09.406Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.258057318Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95402023-09-22T23:28:09.409Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.258096252Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95412023-09-22T23:28:09.412Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.258133798Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1453}
95422023-09-22T23:28:09.415ZLoading write count information from file "/tmp/test_repair_verify.data"
95432023-09-22T23:28:09.418ZRun Repair workload
95442023-09-22T23:28:09.421Z01/30 Write block 591 len 28672 data: 6 9 10 8 8 8 4
95452023-09-22T23:28:09.423Z02/30 Read block 282 len 40960
95462023-09-22T23:28:09.426Z03/30 Flush
95472023-09-22T23:28:09.430Z04/30 Flush
95482023-09-22T23:28:09.433Z05/30 Write block 148 len 20480 data: 5 6 6 7 6
95492023-09-22T23:28:09.436Z06/30 Read block 485 len 20480
95502023-09-22T23:28:09.439Z07/30 Read block 381 len 40960
95512023-09-22T23:28:09.441Z08/30 Read block 76 len 16384
95522023-09-22T23:28:09.444Z09/30 Read block 361 len 28672
95532023-09-22T23:28:09.447Z10/30 Flush
95542023-09-22T23:28:09.450Z11/30 Write block 528 len 24576 data: 7 7 5 6 7 8
95552023-09-22T23:28:09.452Z12/30 Read block 24 len 8192
95562023-09-22T23:28:09.455Z13/30 Write block 236 len 40960 data: 3 4 5 6 6 8 7 6 4 4
95572023-09-22T23:28:09.458Z14/30 Flush
95582023-09-22T23:28:09.461Z15/30 Flush
95592023-09-22T23:28:09.464Z16/30 Write block 292 len 4096 data: 7
95602023-09-22T23:28:09.467Z17/30 Read block 271 len 40960
95612023-09-22T23:28:09.469Z18/30 Write block 562 len 32768 data: 4 4 7 6 7 9 10 10
95622023-09-22T23:28:09.472Z19/30 Read block 335 len 12288
95632023-09-22T23:28:09.475Z20/30 Write block 251 len 36864 data: 4 7 6 7 6 6 6 7 6
95642023-09-22T23:28:09.478Z21/30 Read block 51 len 36864
95652023-09-22T23:28:09.481Z22/30 Read block 276 len 12288
95662023-09-22T23:28:09.484Z23/30 Write block 86 len 24576 data: 6 7 6 6 6 6
95672023-09-22T23:28:09.487Z24/30 Write block 533 len 28672 data: 9 7 7 7 9 9 9
95682023-09-22T23:28:09.490Z25/30 Read block 587 len 4096
95692023-09-22T23:28:09.493Z26/30 Read block 576 len 16384
95702023-09-22T23:28:09.499Z27/30 Read block 253 len 40960
95712023-09-22T23:28:09.502Z28/30 Read block 67 len 40960
95722023-09-22T23:28:09.505Z29/30 Read block 536 len 28672
95732023-09-22T23:28:09.508Z30/30 Read block 561 len 4096
95742023-09-22T23:28:09.919Z----------------------------------------------------------------
95752023-09-22T23:28:09.922Z Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30
95762023-09-22T23:28:09.925ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
95772023-09-22T23:28:09.928Z 1 Acked 1000 Write 7 Done Done Done false
95782023-09-22T23:28:09.931Z 2 Acked 1001 Read 10 Sent Done Done false
95792023-09-22T23:28:09.933Z 3 Acked 1002 Flush 0 Sent Done Done false
95802023-09-22T23:28:09.936Z 4 Acked 1003 Flush 0 Sent Done Done false
95812023-09-22T23:28:09.939Z 5 Acked 1004 Write 5 Sent Done Done false
95822023-09-22T23:28:09.942Z 6 Acked 1005 Read 5 Sent Done Done false
95832023-09-22T23:28:09.945Z 7 Acked 1006 Read 10 Sent Done Done false
95842023-09-22T23:28:09.948Z 8 Acked 1007 Read 4 Sent Done Done false
95852023-09-22T23:28:09.951Z 9 Acked 1008 Read 7 Sent Done Done false
95862023-09-22T23:28:09.954Z 10 Acked 1009 Flush 0 Sent Done Done false
95872023-09-22T23:28:09.957Z 11 Acked 1010 Write 6 Sent Done Done false
95882023-09-22T23:28:09.960Z 12 Acked 1011 Read 2 Sent Done Done false
95892023-09-22T23:28:09.963Z 13 Acked 1012 Write 10 Sent Done Done false
95902023-09-22T23:28:09.966Z 14 Acked 1013 Flush 0 Sent Done Done false
95912023-09-22T23:28:09.969Z 15 Acked 1014 Flush 0 Sent Done Done false
95922023-09-22T23:28:09.972Z 16 Acked 1015 Write 1 Sent Done Done false
95932023-09-22T23:28:09.975Z 17 Acked 1016 Read 10 Sent Done Done false
95942023-09-22T23:28:09.978Z 18 Acked 1017 Write 8 Sent Done Done false
95952023-09-22T23:28:09.981Z 19 Acked 1018 Read 3 Sent Done Done false
95962023-09-22T23:28:09.984Z 20 Acked 1019 Write 9 Sent Done Done false
95972023-09-22T23:28:09.987Z 21 Acked 1020 Read 9 Sent Done Done false
95982023-09-22T23:28:09.990Z 22 Acked 1021 Read 3 Sent Done Done false
95992023-09-22T23:28:09.993Z 23 Acked 1022 Write 6 Sent Done Done false
96002023-09-22T23:28:09.996Z 24 Acked 1023 Write 7 Sent Done Done false
96012023-09-22T23:28:09.999Z 25 Acked 1024 Read 1 Sent Done Done false
96022023-09-22T23:28:10.002Z 26 Acked 1025 Read 4 Sent Done Done false
96032023-09-22T23:28:10.005Z 27 Acked 1026 Read 10 Sent Done Done false
96042023-09-22T23:28:10.008Z 28 Acked 1027 Read 10 Sent Done Done false
96052023-09-22T23:28:10.011Z 29 Acked 1028 Read 7 Sent Done Done false
96062023-09-22T23:28:10.014Z 30 Acked 1029 Read 1 Sent Sent Done false
96072023-09-22T23:28:10.017Z STATES DS:0 DS:1 DS:2 TOTAL
96082023-09-22T23:28:10.020Z New 0 0 0 0
96092023-09-22T23:28:10.429Z Sent 29 1 0 30
96102023-09-22T23:28:10.432Z Done 1 29 30 60
96112023-09-22T23:28:10.435Z Skipped 0 0 0 0
96122023-09-22T23:28:10.438Z Error 0 0 0 0
96132023-09-22T23:28:10.441ZLast Flush: 0 1014 1014
96142023-09-22T23:28:10.443ZDownstairs last five completed:
96152023-09-22T23:28:10.446ZUpstairs last five completed: 30 29 28 27 26 25
96162023-09-22T23:28:10.450ZWrote out file "/tmp/test_repair_verify.data"
96172023-09-22T23:28:10.457Z
96182023-09-22T23:28:10.460ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
96192023-09-22T23:28:10.463Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.392226267Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1454}
96202023-09-22T23:28:10.465Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.392488988Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1454}
96212023-09-22T23:28:10.468Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.392511969Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1454}
96222023-09-22T23:28:10.471Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.425568021Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1454}
96232023-09-22T23:28:10.474Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.425602389Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1454}
96242023-09-22T23:28:10.477ZError: Difference in extent metadata found!
96252023-09-22T23:28:10.480Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.425621983Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1454}
96262023-09-22T23:28:10.483Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.443709875Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1454}
96272023-09-22T23:28:10.486Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.443742253Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1454}
96282023-09-22T23:28:10.489Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:09.443762179Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1454}
96292023-09-22T23:28:10.491ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
96302023-09-22T23:28:10.494Z 0 000-019 60 60 60 90 90 90 F F F
96312023-09-22T23:28:10.497Z 1 020-039 66 66 66 94 94 94 F F F
96322023-09-22T23:28:10.500Z 2 040-059 62 62 62 92 92 92 F F F
96332023-09-22T23:28:10.503Z 3 060-079 69 69 69 100 100 100 F F F
96342023-09-22T23:28:10.506Z 4 080-099 69 69 69 100 100 100 F T T <---
96352023-09-22T23:28:10.509Z 5 100-119 70 70 70 103 103 103 F F F
96362023-09-22T23:28:10.511Z 6 120-139 69 69 69 100 100 100 F F F
96372023-09-22T23:28:10.514Z 7 140-159 70 72 72 102 107 107 F F F <---
96382023-09-22T23:28:10.517Z 8 160-179 69 69 69 100 100 100 F F F
96392023-09-22T23:28:10.520Z 9 180-199 71 71 71 104 104 104 F F F
96402023-09-22T23:28:10.523Z 10 200-219 71 71 71 104 104 104 F F F
96412023-09-22T23:28:10.525Z 11 220-239 66 72 72 96 108 108 F F F <---
96422023-09-22T23:28:10.528Z 12 240-259 69 72 72 100 108 108 F T T <---
96432023-09-22T23:28:10.531Z 13 260-279 69 69 69 100 100 100 F F F
96442023-09-22T23:28:10.534Z 14 280-299 61 61 61 91 91 91 F T T <---
96452023-09-22T23:28:10.537Z 15 300-319 69 69 69 100 100 100 F F F
96462023-09-22T23:28:10.539Z 16 320-339 69 69 69 100 100 100 F F F
96472023-09-22T23:28:10.542Z 17 340-359 66 66 66 95 95 95 F F F
96482023-09-22T23:28:10.545Z 18 360-379 71 71 71 104 104 104 F F F
96492023-09-22T23:28:10.548Z 19 380-399 71 71 71 104 104 104 F F F
96502023-09-22T23:28:10.551Z 20 400-419 70 70 70 101 101 101 F F F
96512023-09-22T23:28:10.553Z 21 420-439 70 70 70 101 101 101 F F F
96522023-09-22T23:28:10.556Z 22 440-459 71 71 71 104 104 104 F F F
96532023-09-22T23:28:10.559Z 23 460-479 71 71 71 104 104 104 F F F
96542023-09-22T23:28:10.562Z 24 480-499 70 70 70 101 101 101 F F F
96552023-09-22T23:28:10.564Z 25 500-519 69 69 69 100 100 100 F F F
96562023-09-22T23:28:10.568Z 26 520-539 69 72 72 100 108 108 F T T <---
96572023-09-22T23:28:10.571Z 27 540-559 71 71 71 104 104 104 F F F
96582023-09-22T23:28:10.574Z 28 560-579 71 71 71 104 104 104 F T T <---
96592023-09-22T23:28:10.576Z 29 580-599 59 72 72 89 105 105 T F F <---
96602023-09-22T23:28:10.579ZMax gen: 72, Max flush: 108
96612023-09-22T23:28:10.582ZA Difference in extent metadata is expected here
96622023-09-22T23:28:10.585ZOn loop 36
96632023-09-22T23:28:10.591Z
96642023-09-22T23:28:10.594ZVerifying data now
96652023-09-22T23:28:15.477ZNo change after verify
96662023-09-22T23:28:15.479ZLoop: 36 Downstairs dump after verify (and repair):
96672023-09-22T23:28:15.485Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.483754344Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1459}
96682023-09-22T23:28:15.488Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.484072143Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1459}
96692023-09-22T23:28:15.492Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.484096931Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1459}
96702023-09-22T23:28:15.504Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.503112774Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1459}
96712023-09-22T23:28:15.507Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.503165043Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1459}
96722023-09-22T23:28:15.511Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.503185468Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1459}
96732023-09-22T23:28:15.514Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.514189317Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1459}
96742023-09-22T23:28:15.517Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.514241218Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1459}
96752023-09-22T23:28:15.520Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.514261492Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1459}
96762023-09-22T23:28:15.525ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
96772023-09-22T23:28:15.529Z 0 000-019 60 60 60 90 90 90 F F F
96782023-09-22T23:28:15.533Z 1 020-039 66 66 66 94 94 94 F F F
96792023-09-22T23:28:15.535Z 2 040-059 62 62 62 92 92 92 F F F
96802023-09-22T23:28:15.538Z 3 060-079 69 69 69 100 100 100 F F F
96812023-09-22T23:28:15.541Z 4 080-099 73 73 73 109 109 109 F F F
96822023-09-22T23:28:15.545Z 5 100-119 70 70 70 103 103 103 F F F
96832023-09-22T23:28:15.548Z 6 120-139 69 69 69 100 100 100 F F F
96842023-09-22T23:28:15.552Z 7 140-159 72 72 72 107 107 107 F F F
96852023-09-22T23:28:15.555Z 8 160-179 69 69 69 100 100 100 F F F
96862023-09-22T23:28:15.558Z 9 180-199 71 71 71 104 104 104 F F F
96872023-09-22T23:28:15.562Z 10 200-219 71 71 71 104 104 104 F F F
96882023-09-22T23:28:15.565Z 11 220-239 72 72 72 108 108 108 F F F
96892023-09-22T23:28:15.567Z 12 240-259 73 73 73 109 109 109 F F F
96902023-09-22T23:28:15.570Z 13 260-279 69 69 69 100 100 100 F F F
96912023-09-22T23:28:15.573Z 14 280-299 73 73 73 109 109 109 F F F
96922023-09-22T23:28:15.576Z 15 300-319 69 69 69 100 100 100 F F F
96932023-09-22T23:28:15.579Z 16 320-339 69 69 69 100 100 100 F F F
96942023-09-22T23:28:15.582Z 17 340-359 66 66 66 95 95 95 F F F
96952023-09-22T23:28:15.585Z 18 360-379 71 71 71 104 104 104 F F F
96962023-09-22T23:28:15.588Z 19 380-399 71 71 71 104 104 104 F F F
96972023-09-22T23:28:15.591Z 20 400-419 70 70 70 101 101 101 F F F
96982023-09-22T23:28:15.594Z 21 420-439 70 70 70 101 101 101 F F F
96992023-09-22T23:28:15.597Z 22 440-459 71 71 71 104 104 104 F F F
97002023-09-22T23:28:15.601Z 23 460-479 71 71 71 104 104 104 F F F
97012023-09-22T23:28:15.604Z 24 480-499 70 70 70 101 101 101 F F F
97022023-09-22T23:28:15.606Z 25 500-519 69 69 69 100 100 100 F F F
97032023-09-22T23:28:15.609Z 26 520-539 73 73 73 109 109 109 F F F
97042023-09-22T23:28:15.612Z 27 540-559 71 71 71 104 104 104 F F F
97052023-09-22T23:28:15.615Z 28 560-579 73 73 73 109 109 109 F F F
97062023-09-22T23:28:15.618Z 29 580-599 72 72 72 105 105 105 F F F
97072023-09-22T23:28:15.621ZMax gen: 73, Max flush: 109
97082023-09-22T23:28:15.624Z
97092023-09-22T23:28:15.627ZBegin loop 37
97102023-09-22T23:28:15.630ZDownstairs to restart: 2
97112023-09-22T23:28:15.633Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.563154178Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97122023-09-22T23:28:15.637Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.56355622Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97132023-09-22T23:28:15.640Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.563608248Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97142023-09-22T23:28:15.643Z{"msg":"Crucible stats registered with UUID: 11a1ef4d-0512-4cd7-b423-6e6f5e5d4a20","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.563633285Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97152023-09-22T23:28:15.646Z{"msg":"Crucible 11a1ef4d-0512-4cd7-b423-6e6f5e5d4a20 has session id: 94e0e47e-8760-4f0a-974f-3845884f13aa","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.563673772Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97162023-09-22T23:28:15.648ZCrucible runtime is spawned
97172023-09-22T23:28:16.058ZThe guest has requested activation with gen:74
97182023-09-22T23:28:16.062Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.564055288Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461,"task":"up_listen"}
97192023-09-22T23:28:16.064Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.564155058Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97202023-09-22T23:28:16.067Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.564220145Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97212023-09-22T23:28:16.070Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.56428631Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461,"looper":"1"}
97222023-09-22T23:28:16.073Z{"msg":"[1] 11a1ef4d-0512-4cd7-b423-6e6f5e5d4a20 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.564379912Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461,"looper":"1"}
97232023-09-22T23:28:16.076Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.564481908Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461,"looper":"2"}
97242023-09-22T23:28:16.079Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.5647219Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461,"looper":"0"}
97252023-09-22T23:28:16.082Z{"msg":"[0] 11a1ef4d-0512-4cd7-b423-6e6f5e5d4a20 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.564961885Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461,"looper":"0"}
97262023-09-22T23:28:16.085Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.565108606Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97272023-09-22T23:28:16.088Z{"msg":"11a1ef4d-0512-4cd7-b423-6e6f5e5d4a20 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.565160891Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97282023-09-22T23:28:16.091Z{"msg":"Set desired generation to :74","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.565183388Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97292023-09-22T23:28:16.094Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.565449158Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97302023-09-22T23:28:16.098Z{"msg":"[1] 11a1ef4d-0512-4cd7-b423-6e6f5e5d4a20 (991852ce-d447-4810-8e91-de4488a2426a) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.565756014Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97312023-09-22T23:28:16.101Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.56582515Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97322023-09-22T23:28:16.103Z{"msg":"[1] client is_active_req TRUE, promote! session 991852ce-d447-4810-8e91-de4488a2426a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.565895608Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97332023-09-22T23:28:16.106Z{"msg":"[0] 11a1ef4d-0512-4cd7-b423-6e6f5e5d4a20 (991852ce-d447-4810-8e91-de4488a2426a) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.565973611Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97342023-09-22T23:28:16.109Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.566040491Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97352023-09-22T23:28:16.116Z{"msg":"[0] client is_active_req TRUE, promote! session 991852ce-d447-4810-8e91-de4488a2426a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.566097407Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97362023-09-22T23:28:16.119Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.566468302Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97372023-09-22T23:28:16.122Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.566535148Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97382023-09-22T23:28:16.125Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.566604046Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97392023-09-22T23:28:16.128Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.566689483Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97402023-09-22T23:28:16.130Z{"msg":"11a1ef4d-0512-4cd7-b423-6e6f5e5d4a20 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.566761222Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97412023-09-22T23:28:16.133Z{"msg":"11a1ef4d-0512-4cd7-b423-6e6f5e5d4a20 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.566833705Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97422023-09-22T23:28:16.136Z{"msg":"[1] 11a1ef4d-0512-4cd7-b423-6e6f5e5d4a20 (991852ce-d447-4810-8e91-de4488a2426a) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.568650213Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97432023-09-22T23:28:16.141Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.568710172Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97442023-09-22T23:28:16.145Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:28:15.568752472Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97452023-09-22T23:28:16.148Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.568832694Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97462023-09-22T23:28:16.151Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.568875968Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97472023-09-22T23:28:16.153Z{"msg":"11a1ef4d-0512-4cd7-b423-6e6f5e5d4a20 WaitActive WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.568915703Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97482023-09-22T23:28:16.156Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.568954125Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97492023-09-22T23:28:16.160Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.568992005Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97502023-09-22T23:28:16.163Z{"msg":"[0] 11a1ef4d-0512-4cd7-b423-6e6f5e5d4a20 (991852ce-d447-4810-8e91-de4488a2426a) WaitActive WaitQuorum New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.569618207Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97512023-09-22T23:28:16.166Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.569677591Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97522023-09-22T23:28:16.169Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:28:15.569721122Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97532023-09-22T23:28:16.172Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.569767198Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97542023-09-22T23:28:16.175Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.569808367Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97552023-09-22T23:28:16.178Z{"msg":"11a1ef4d-0512-4cd7-b423-6e6f5e5d4a20 WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.569829292Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97562023-09-22T23:28:16.181Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.569846948Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97572023-09-22T23:28:16.183Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:15.569864321Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97582023-09-22T23:28:16.566Z{"msg":"[2] 11a1ef4d-0512-4cd7-b423-6e6f5e5d4a20 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.56584401Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461,"looper":"2"}
97592023-09-22T23:28:16.569Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.565932733Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97602023-09-22T23:28:16.572Z{"msg":"[2] 11a1ef4d-0512-4cd7-b423-6e6f5e5d4a20 (991852ce-d447-4810-8e91-de4488a2426a) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.567011967Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97612023-09-22T23:28:16.575Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.567075489Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97622023-09-22T23:28:16.578Z{"msg":"[2] client is_active_req TRUE, promote! session 991852ce-d447-4810-8e91-de4488a2426a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.567119277Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97632023-09-22T23:28:16.581Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.567713079Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97642023-09-22T23:28:16.584Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.567769404Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97652023-09-22T23:28:16.587Z{"msg":"11a1ef4d-0512-4cd7-b423-6e6f5e5d4a20 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.567811359Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97662023-09-22T23:28:16.590Z{"msg":"[2] 11a1ef4d-0512-4cd7-b423-6e6f5e5d4a20 (991852ce-d447-4810-8e91-de4488a2426a) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.570148163Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97672023-09-22T23:28:16.593Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.570222033Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97682023-09-22T23:28:16.596Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:28:16.570288229Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97692023-09-22T23:28:16.599Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.570348471Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97702023-09-22T23:28:16.602Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.57039254Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97712023-09-22T23:28:16.604Z{"msg":"11a1ef4d-0512-4cd7-b423-6e6f5e5d4a20 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.570431917Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97722023-09-22T23:28:16.608Z{"msg":"[0]R flush_numbers[0..12]: [90, 94, 92, 100, 109, 103, 100, 107, 100, 104, 104, 108]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.570471726Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97732023-09-22T23:28:16.611Z{"msg":"[0]R generation[0..12]: [60, 66, 62, 69, 73, 70, 69, 72, 69, 71, 71, 72]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.57051134Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97742023-09-22T23:28:16.614Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:74
97752023-09-22T23:28:16.617ZWait for a query_work_queue command to finish before sending IO
97762023-09-22T23:28:16.620Z,"time":"2023-09-22T23:28:16.570551157Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97772023-09-22T23:28:16.622Z{"msg":"[1]R flush_numbers[0..12]: [90, 94, 92, 100, 109, 103, 100, 107, 100, 104, 104, 108]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.570771352Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97782023-09-22T23:28:16.625Z{"msg":"[1]R generation[0..12]: [60, 66, 62, 69, 73, 70, 69, 72, 69, 71, 71, 72]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.570845826Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97792023-09-22T23:28:16.628Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.570974802Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97802023-09-22T23:28:16.631Z{"msg":"[2]R flush_numbers[0..12]: [90, 94, 92, 100, 109, 103, 100, 107, 100, 104, 104, 108]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.571039926Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97812023-09-22T23:28:16.633Z{"msg":"[2]R generation[0..12]: [60, 66, 62, 69, 73, 70, 69, 72, 69, 71, 71, 72]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.571094569Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97822023-09-22T23:28:16.636Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.571138243Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97832023-09-22T23:28:16.639Z{"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.571159323Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97842023-09-22T23:28:16.642Z{"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.571211563Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97852023-09-22T23:28:16.644Z{"msg":"Next flush: 110","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.571248158Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97862023-09-22T23:28:16.647Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.571280414Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97872023-09-22T23:28:16.650Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.57137515Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97882023-09-22T23:28:16.654Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.571419646Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97892023-09-22T23:28:16.657Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.571462668Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97902023-09-22T23:28:16.660Z{"msg":"11a1ef4d-0512-4cd7-b423-6e6f5e5d4a20 is now active with session: 991852ce-d447-4810-8e91-de4488a2426a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.571526671Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97912023-09-22T23:28:16.663Z{"msg":"11a1ef4d-0512-4cd7-b423-6e6f5e5d4a20 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.571594357Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97922023-09-22T23:28:16.666Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.571630793Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97932023-09-22T23:28:16.669Z{"msg":"Set check for repair","v":0Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
97942023-09-22T23:28:16.671Z,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.571753301Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97952023-09-22T23:28:16.674Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.571824662Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97962023-09-22T23:28:16.677Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.571890272Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97972023-09-22T23:28:16.680Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.571951524Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97982023-09-22T23:28:16.683Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.572016464Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
97992023-09-22T23:28:16.686Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.572080468Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
98002023-09-22T23:28:16.689Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.572141735Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
98012023-09-22T23:28:17.098Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.572207443Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
98022023-09-22T23:28:17.102Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.572274018Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
98032023-09-22T23:28:17.105Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.572334852Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1461}
98042023-09-22T23:28:17.108ZLoading write count information from file "/tmp/test_repair_verify.data"
98052023-09-22T23:28:17.112ZRun Repair workload
98062023-09-22T23:28:17.116Z01/30 Write block 516 len 36864 data: 10 10 10 8 7 5 6 6 5
98072023-09-22T23:28:17.119Z02/30 Read block 24 len 8192
98082023-09-22T23:28:17.122Z03/30 Write block 80 len 12288 data: 10 12 11
98092023-09-22T23:28:17.125Z04/30 Read block 313 len 4096
98102023-09-22T23:28:17.130Z05/30 Read block 512 len 32768
98112023-09-22T23:28:17.135Z06/30 Read block 351 len 4096
98122023-09-22T23:28:17.138Z07/30 Write block 510 len 12288 data: 5 4 7
98132023-09-22T23:28:17.147Z08/30 Flush
98142023-09-22T23:28:17.150Z09/30 Write block 276 len 4096 data: 12
98152023-09-22T23:28:17.153Z10/30 Read block 28 len 12288
98162023-09-22T23:28:17.156Z11/30 Write block 87 len 12288 data: 8 7 7
98172023-09-22T23:28:17.160Z12/30 Read block 267 len 24576
98182023-09-22T23:28:17.163Z13/30 Write block 31 len 16384 data: 5 4 3 6
98192023-09-22T23:28:17.166Z14/30 Write block 538 len 8192 data: 10 10
98202023-09-22T23:28:17.169Z15/30 Read block 442 len 8192
98212023-09-22T23:28:17.172Z16/30 Write block 79 len 4096 data: 11
98222023-09-22T23:28:17.176Z17/30 Read block 209 len 24576
98232023-09-22T23:28:17.179Z18/30 Write block 575 len 12288 data: 4 4 5
98242023-09-22T23:28:17.182Z19/30 Read block 240 len 8192
98252023-09-22T23:28:17.184Z20/30 Read block 528 len 40960
98262023-09-22T23:28:17.188Z21/30 Flush
98272023-09-22T23:28:17.194Z22/30 Write block 291 len 12288 data: 8 8 7
98282023-09-22T23:28:17.198Z23/30 Write block 486 len 28672 data: 8 6 9 11 10 11 10
98292023-09-22T23:28:17.200Z24/30 Read block 514 len 36864
98302023-09-22T23:28:17.204Z25/30 Read block 469 len 36864
98312023-09-22T23:28:17.207Z26/30 Read block 558 len 28672
98322023-09-22T23:28:17.210Z27/30 Write block 213 len 20480 data: 9 7 7 6 6
98332023-09-22T23:28:17.213Z28/30 Read block 473 len 40960
98342023-09-22T23:28:17.216Z29/30 Read block 273 len 8192
98352023-09-22T23:28:17.219Z30/30 Write block 442 len 16384 data: 6 8 9 9
98362023-09-22T23:28:17.222Z----------------------------------------------------------------
98372023-09-22T23:28:17.224Z Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30
98382023-09-22T23:28:17.227ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
98392023-09-22T23:28:17.231Z 1 Acked 1000 Write 9 Done Done Sent false
98402023-09-22T23:28:17.234Z 2 Acked 1001 Read 2 Done Done Sent false
98412023-09-22T23:28:17.237Z 3 Acked 1002 Write 3 Done Done Sent false
98422023-09-22T23:28:17.240Z 4 Acked 1003 Read 1 Done Done Sent false
98432023-09-22T23:28:17.243Z 5 Acked 1004 Read 8 Done Done Sent false
98442023-09-22T23:28:17.246Z 6 Acked 1005 Read 1 Done Done Sent false
98452023-09-22T23:28:17.249Z 7 Acked 1006 Write 3 Done Done Sent false
98462023-09-22T23:28:17.252Z 8 Acked 1007 Flush 0 Done Done Sent false
98472023-09-22T23:28:17.256Z 9 Acked 1008 Write 1 Done Done Sent false
98482023-09-22T23:28:17.259Z 10 Acked 1009 Read 3 Done Done Sent false
98492023-09-22T23:28:17.262Z 11 Acked 1010 Write 3 Done Done Sent false
98502023-09-22T23:28:17.265Z 12 Acked 1011 Read 6 Done Done Sent false
98512023-09-22T23:28:17.267Z 13 Acked 1012 Write 4 Done Done Sent false
98522023-09-22T23:28:17.271Z 14 Acked 1013 Write 2 Done Done Sent false
98532023-09-22T23:28:17.274Z 15 Acked 1014 Read 2 Done Done Sent false
98542023-09-22T23:28:17.277Z 16 Acked 1015 Write 1 Done Done Sent false
98552023-09-22T23:28:17.689Z 17 Acked 1016 Read 6 Done Done Sent false
98562023-09-22T23:28:17.692Z 18 Acked 1017 Write 3 Done Done Sent false
98572023-09-22T23:28:17.695Z 19 Acked 1018 Read 2 Done Done Sent false
98582023-09-22T23:28:17.698Z 20 Acked 1019 Read 10 Done Done Sent false
98592023-09-22T23:28:17.701Z 21 Acked 1020 Flush 0 Done Done Sent false
98602023-09-22T23:28:17.704Z 22 Acked 1021 Write 3 Done Done Sent false
98612023-09-22T23:28:17.706Z 23 Acked 1022 Write 7 Done Done Sent false
98622023-09-22T23:28:17.710Z 24 Acked 1023 Read 9 Done Done Sent false
98632023-09-22T23:28:17.713Z 25 Acked 1024 Read 9 Done Done Sent false
98642023-09-22T23:28:17.716Z 26 Acked 1025 Read 7 Done Done Sent false
98652023-09-22T23:28:17.719Z 27 Acked 1026 Write 5 Done Done Sent false
98662023-09-22T23:28:17.722Z 28 Acked 1027 Read 10 Done Done Sent false
98672023-09-22T23:28:17.725ZError: Difference in extent metadata found!
98682023-09-22T23:28:17.728Z 29 Acked 1028 Read 2 Done Done Sent false
98692023-09-22T23:28:17.731Z 30 Acked 1029 Write 4 Sent Sent Sent false
98702023-09-22T23:28:17.734Z STATES DS:0 DS:1 DS:2 TOTAL
98712023-09-22T23:28:17.737Z New 0 0 0 0
98722023-09-22T23:28:17.740Z Sent 1 1 30 32
98732023-09-22T23:28:17.743Z Done 29 29 0 58
98742023-09-22T23:28:17.746Z Skipped 0 0 0 0
98752023-09-22T23:28:17.749Z Error 0 0 0 0
98762023-09-22T23:28:17.752ZLast Flush: 1020 1020 0
98772023-09-22T23:28:17.755ZDownstairs last five completed:
98782023-09-22T23:28:17.759ZUpstairs last five completed: 30 29 28 27 26 25
98792023-09-22T23:28:17.762ZWrote out file "/tmp/test_repair_verify.data"
98802023-09-22T23:28:17.765Z
98812023-09-22T23:28:17.768ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
98822023-09-22T23:28:17.771Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.698039689Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1462}
98832023-09-22T23:28:17.773Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.698379262Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1462}
98842023-09-22T23:28:17.776Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.69840265Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1462}
98852023-09-22T23:28:17.780Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.723770239Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1462}
98862023-09-22T23:28:17.783Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.723799199Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1462}
98872023-09-22T23:28:17.785Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.723817846Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1462}
98882023-09-22T23:28:17.788Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.739965063Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1462}
98892023-09-22T23:28:17.791Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.739993132Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1462}
98902023-09-22T23:28:17.794Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:16.740011562Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1462}
98912023-09-22T23:28:17.797ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
98922023-09-22T23:28:17.800Z 0 000-019 60 60 60 90 90 90 F F F
98932023-09-22T23:28:17.803Z 1 020-039 74 74 66 111 111 94 F F F <---
98942023-09-22T23:28:17.805Z 2 040-059 62 62 62 92 92 92 F F F
98952023-09-22T23:28:17.809Z 3 060-079 74 74 69 111 111 100 F F F <---
98962023-09-22T23:28:17.812Z 4 080-099 74 74 73 111 111 109 F F F <---
98972023-09-22T23:28:17.815Z 5 100-119 70 70 70 103 103 103 F F F
98982023-09-22T23:28:17.818Z 6 120-139 69 69 69 100 100 100 F F F
98992023-09-22T23:28:17.821Z 7 140-159 72 72 72 107 107 107 F F F
99002023-09-22T23:28:17.824Z 8 160-179 69 69 69 100 100 100 F F F
99012023-09-22T23:28:17.828Z 9 180-199 71 71 71 104 104 104 F F F
99022023-09-22T23:28:17.830Z 10 200-219 71 71 71 104 104 104 T T F <---
99032023-09-22T23:28:17.833Z 11 220-239 72 72 72 108 108 108 F F F
99042023-09-22T23:28:17.836Z 12 240-259 73 73 73 109 109 109 F F F
99052023-09-22T23:28:17.840Z 13 260-279 74 74 69 111 111 100 F F F <---
99062023-09-22T23:28:17.842Z 14 280-299 73 73 73 109 109 109 T T F <---
99072023-09-22T23:28:17.845Z 15 300-319 69 69 69 100 100 100 F F F
99082023-09-22T23:28:17.848Z 16 320-339 69 69 69 100 100 100 F F F
99092023-09-22T23:28:17.851Z 17 340-359 66 66 66 95 95 95 F F F
99102023-09-22T23:28:17.853Z 18 360-379 71 71 71 104 104 104 F F F
99112023-09-22T23:28:17.857Z 19 380-399 71 71 71 104 104 104 F F F
99122023-09-22T23:28:17.860Z 20 400-419 70 70 70 101 101 101 F F F
99132023-09-22T23:28:17.863Z 21 420-439 70 70 70 101 101 101 F F F
99142023-09-22T23:28:17.866Z 22 440-459 71 71 71 104 104 104 T T F <---
99152023-09-22T23:28:17.869Z 23 460-479 71 71 71 104 104 104 F F F
99162023-09-22T23:28:17.871Z 24 480-499 70 70 70 101 101 101 T T F <---
99172023-09-22T23:28:17.874Z 25 500-519 74 74 69 110 110 100 F F F <---
99182023-09-22T23:28:17.877Z 26 520-539 74 74 73 111 111 109 F F F <---
99192023-09-22T23:28:17.880Z 27 540-559 71 71 71 104 104 104 F F F
99202023-09-22T23:28:17.883Z 28 560-579 74 74 73 111 111 109 F F F <---
99212023-09-22T23:28:17.887Z 29 580-599 72 72 72 105 105 105 F F F
99222023-09-22T23:28:17.891ZMax gen: 74, Max flush: 111
99232023-09-22T23:28:17.894ZA Difference in extent metadata is expected here
99242023-09-22T23:28:17.896ZOn loop 37
99252023-09-22T23:28:17.899Z
99262023-09-22T23:28:17.902ZVerifying data now
99272023-09-22T23:28:22.999ZNo change after verify
99282023-09-22T23:28:23.003ZLoop: 37 Downstairs dump after verify (and repair):
99292023-09-22T23:28:23.019Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.018046931Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1467}
99302023-09-22T23:28:23.022Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.018410089Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1467}
99312023-09-22T23:28:23.025Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.018435001Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1467}
99322023-09-22T23:28:23.037Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.036922096Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1467}
99332023-09-22T23:28:23.041Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.03697987Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1467}
99342023-09-22T23:28:23.043Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.037001832Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1467}
99352023-09-22T23:28:23.049Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.047763907Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1467}
99362023-09-22T23:28:23.052Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.047816633Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1467}
99372023-09-22T23:28:23.055Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.047837262Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1467}
99382023-09-22T23:28:23.059ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
99392023-09-22T23:28:23.064Z 0 000-019 60 60 60 90 90 90 F F F
99402023-09-22T23:28:23.067Z 1 020-039 74 74 74 111 111 111 F F F
99412023-09-22T23:28:23.070Z 2 040-059 62 62 62 92 92 92 F F F
99422023-09-22T23:28:23.073Z 3 060-079 74 74 74 111 111 111 F F F
99432023-09-22T23:28:23.076Z 4 080-099 74 74 74 111 111 111 F F F
99442023-09-22T23:28:23.079Z 5 100-119 70 70 70 103 103 103 F F F
99452023-09-22T23:28:23.082Z 6 120-139 69 69 69 100 100 100 F F F
99462023-09-22T23:28:23.085Z 7 140-159 72 72 72 107 107 107 F F F
99472023-09-22T23:28:23.088Z 8 160-179 69 69 69 100 100 100 F F F
99482023-09-22T23:28:23.091Z 9 180-199 71 71 71 104 104 104 F F F
99492023-09-22T23:28:23.094Z 10 200-219 75 75 75 112 112 112 F F F
99502023-09-22T23:28:23.097Z 11 220-239 72 72 72 108 108 108 F F F
99512023-09-22T23:28:23.100Z 12 240-259 73 73 73 109 109 109 F F F
99522023-09-22T23:28:23.103Z 13 260-279 74 74 74 111 111 111 F F F
99532023-09-22T23:28:23.106Z 14 280-299 75 75 75 112 112 112 F F F
99542023-09-22T23:28:23.109Z 15 300-319 69 69 69 100 100 100 F F F
99552023-09-22T23:28:23.112Z 16 320-339 69 69 69 100 100 100 F F F
99562023-09-22T23:28:23.114Z 17 340-359 66 66 66 95 95 95 F F F
99572023-09-22T23:28:23.117Z 18 360-379 71 71 71 104 104 104 F F F
99582023-09-22T23:28:23.120Z 19 380-399 71 71 71 104 104 104 F F F
99592023-09-22T23:28:23.123Z 20 400-419 70 70 70 101 101 101 F F F
99602023-09-22T23:28:23.125Z 21 420-439 70 70 70 101 101 101 F F F
99612023-09-22T23:28:23.128Z 22 440-459 75 75 75 112 112 112 F F F
99622023-09-22T23:28:23.131Z 23 460-479 71 71 71 104 104 104 F F F
99632023-09-22T23:28:23.134Z 24 480-499 75 75 75 112 112 112 F F F
99642023-09-22T23:28:23.137Z 25 500-519 74 74 74 110 110 110 F F F
99652023-09-22T23:28:23.140Z 26 520-539 74 74 74 111 111 111 F F F
99662023-09-22T23:28:23.143Z 27 540-559 71 71 71 104 104 104 F F F
99672023-09-22T23:28:23.146Z 28 560-579 74 74 74 111 111 111 F F F
99682023-09-22T23:28:23.149Z 29 580-599 72 72 72 105 105 105 F F F
99692023-09-22T23:28:23.151ZMax gen: 75, Max flush: 112
99702023-09-22T23:28:23.154Z
99712023-09-22T23:28:23.157ZBegin loop 38
99722023-09-22T23:28:23.160ZDownstairs to restart: 1
99732023-09-22T23:28:23.163Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.095732083Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
99742023-09-22T23:28:23.166Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.096103189Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
99752023-09-22T23:28:23.169Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.09615881Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
99762023-09-22T23:28:23.172Z{"msg":"Crucible stats registered with UUID: f1559b4b-06ad-47ec-9b3d-a0691a8f0cf0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.096202178Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
99772023-09-22T23:28:23.175Z{"msg":"Crucible f1559b4b-06ad-47ec-9b3d-a0691a8f0cf0 has session id: 73e3d937-9cd6-4152-993c-a798fdc5d29e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.096233515Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
99782023-09-22T23:28:23.178ZCrucible runtime is spawned
99792023-09-22T23:28:23.181Z{The guest has requested activation with gen:76
99802023-09-22T23:28:23.184Z"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.096526105Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469,"task":"up_listen"}
99812023-09-22T23:28:23.186Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.09658371Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
99822023-09-22T23:28:23.189Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.096633714Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
99832023-09-22T23:28:23.192Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.096655287Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469,"looper":"0"}
99842023-09-22T23:28:23.195Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.096917978Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469,"looper":"1"}
99852023-09-22T23:28:23.198Z{"msg":"[0] f1559b4b-06ad-47ec-9b3d-a0691a8f0cf0 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.097006896Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469,"looper":"0"}
99862023-09-22T23:28:23.201Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.097063696Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469,"looper":"2"}
99872023-09-22T23:28:23.204Z{"msg":"[2] f1559b4b-06ad-47ec-9b3d-a0691a8f0cf0 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.097242835Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469,"looper":"2"}
99882023-09-22T23:28:23.209Z{"msg":"f1559b4b-06ad-47ec-9b3d-a0691a8f0cf0 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.097544701Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
99892023-09-22T23:28:23.212Z{"msg":"Set desired generation to :76","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.097595032Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
99902023-09-22T23:28:23.215Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.097636613Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
99912023-09-22T23:28:23.219Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.09789358Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
99922023-09-22T23:28:23.222Z{"msg":"[0] f1559b4b-06ad-47ec-9b3d-a0691a8f0cf0 (d423e36d-97c8-4f7d-a227-29831a8ea008) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.098174485Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
99932023-09-22T23:28:23.225Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.098278181Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
99942023-09-22T23:28:23.228Z{"msg":"[0] client is_active_req TRUE, promote! session d423e36d-97c8-4f7d-a227-29831a8ea008","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.09833939Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
99952023-09-22T23:28:23.231Z{"msg":"[2] f1559b4b-06ad-47ec-9b3d-a0691a8f0cf0 (d423e36d-97c8-4f7d-a227-29831a8ea008) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.098404962Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
99962023-09-22T23:28:23.234Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.098460113Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
99972023-09-22T23:28:23.237Z{"msg":"[2] client is_active_req TRUE, promote! session d423e36d-97c8-4f7d-a227-29831a8ea008","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.09851711Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
99982023-09-22T23:28:23.240Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.098851795Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
99992023-09-22T23:28:23.244Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.098928729Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100002023-09-22T23:28:23.247Z{"msg":"f1559b4b-06ad-47ec-9b3d-a0691a8f0cf0 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.098983454Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100012023-09-22T23:28:23.250Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.099039941Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100022023-09-22T23:28:23.253Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.099111527Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100032023-09-22T23:28:23.256Z{"msg":"f1559b4b-06ad-47ec-9b3d-a0691a8f0cf0 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.099164601Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100042023-09-22T23:28:23.259Z{"msg":"[0] f1559b4b-06ad-47ec-9b3d-a0691a8f0cf0 (d423e36d-97c8-4f7d-a227-29831a8ea008) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.100926143Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100052023-09-22T23:28:23.263Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.100982771Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100062023-09-22T23:28:23.266Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:28:23.101024366Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100072023-09-22T23:28:23.269Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.101062652Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100082023-09-22T23:28:23.272Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.101101056Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100092023-09-22T23:28:23.275Z{"msg":"f1559b4b-06ad-47ec-9b3d-a0691a8f0cf0 WaitQuorum New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.101139779Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100102023-09-22T23:28:23.278Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.10117788Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100112023-09-22T23:28:23.281Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.10124648Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100122023-09-22T23:28:23.284Z{"msg":"[2] f1559b4b-06ad-47ec-9b3d-a0691a8f0cf0 (d423e36d-97c8-4f7d-a227-29831a8ea008) WaitQuorum New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.102240952Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100132023-09-22T23:28:23.287Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.102300025Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100142023-09-22T23:28:23.290Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:28:23.102341564Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100152023-09-22T23:28:23.293Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.102380617Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100162023-09-22T23:28:23.296Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.102419483Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100172023-09-22T23:28:23.299Z{"msg":"f1559b4b-06ad-47ec-9b3d-a0691a8f0cf0 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.102458077Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100182023-09-22T23:28:23.302Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.102477806Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100192023-09-22T23:28:23.305Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:23.102495212Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100202023-09-22T23:28:24.099Z{"msg":"[1] f1559b4b-06ad-47ec-9b3d-a0691a8f0cf0 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.098439713Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469,"looper":"1"}
100212023-09-22T23:28:24.102Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.098523905Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100222023-09-22T23:28:24.105Z{"msg":"[1] f1559b4b-06ad-47ec-9b3d-a0691a8f0cf0 (d423e36d-97c8-4f7d-a227-29831a8ea008) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.099540139Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100232023-09-22T23:28:24.109Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.09959483Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100242023-09-22T23:28:24.112Z{"msg":"[1] client is_active_req TRUE, promote! session d423e36d-97c8-4f7d-a227-29831a8ea008","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.099618656Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100252023-09-22T23:28:24.115Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.100235679Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100262023-09-22T23:28:24.528Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.10029204Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100272023-09-22T23:28:24.532Z{"msg":"f1559b4b-06ad-47ec-9b3d-a0691a8f0cf0 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.100334966Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100282023-09-22T23:28:24.535Z{"msg":"[1] f1559b4b-06ad-47ec-9b3d-a0691a8f0cf0 (d423e36d-97c8-4f7d-a227-29831a8ea008) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.102694147Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100292023-09-22T23:28:24.538Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.102763896Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100302023-09-22T23:28:24.541Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:28:24.102809965Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100312023-09-22T23:28:24.545Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.102848799Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100322023-09-22T23:28:24.548Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.102887752Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100332023-09-22T23:28:24.551Z{"msg":"f1559b4b-06ad-47ec-9b3d-a0691a8f0cf0 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.102926933Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100342023-09-22T23:28:24.554Z{"msg":"[0]R flush_numbers[0..12]: [90, 111, 92, 111, 111, 103, 100, 107, 100, 104, 112, 108]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.102966348Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100352023-09-22T23:28:24.557Z{"msg":"[0]R generation[0..12]: [60, 74, 62, 74, 74, 70, 69, 72, 69, 71, 75, 72]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.103005775Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100362023-09-22T23:28:24.560Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.103062495Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100372023-09-22T23:28:24.562Z{"msg":"[1]R flush_numbers[0..12]: [90, 111, 92, 111, 111, 103, 100, 107, 100, 104, 112, 108]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.103084195Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100382023-09-22T23:28:24.566Z{"msg":"[1]R generation[0..12]: [60, 74, 62, 74, 74, 70, 69, 72, 69, 71, 75, 72]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.103103141Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100392023-09-22T23:28:24.568Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.103176142Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469The guest has finished waiting for activation with:76
100402023-09-22T23:28:24.571ZWait for a query_work_queue command to finish before sending IO
100412023-09-22T23:28:24.574Z}
100422023-09-22T23:28:24.577Z{"msg":"[2]R flush_numbers[0..12]: [90, 111, 92, 111, 111, 103, 100, 107, 100, 104, 112, 108]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.103345344Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100432023-09-22T23:28:24.580Z{"msg":"[2]R generation[0..12]: [60, 74, 62, 74, 74, 70, 69, 72, 69, 71, 75, 72]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.103395872Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100442023-09-22T23:28:24.583Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.103437384Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100452023-09-22T23:28:24.586Z{"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.10347655Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100462023-09-22T23:28:24.589Z{"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.103515461Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100472023-09-22T23:28:24.592Z{"msg":"Next flush: 113","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.103536349Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100482023-09-22T23:28:24.595Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.103580203Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100492023-09-22T23:28:24.597Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.103621006Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100502023-09-22T23:28:24.600Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.103660466Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100512023-09-22T23:28:24.603Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.103711909Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100522023-09-22T23:28:24.606Z{"msg":"f1559b4b-06ad-47ec-9b3d-a0691a8f0cf0 is now active with session: d423e36d-97c8-4f7d-a227-29831a8ea008","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.103751492Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100532023-09-22T23:28:24.609Z{"msg":"f1559b4b-06ad-47ec-9b3d-a0691a8f0cf0 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.103773244Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100542023-09-22T23:28:24.611Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.103810043Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100552023-09-22T23:28:24.614Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.103848899Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100562023-09-22T23:28:24.617Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.103896916Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100572023-09-22T23:28:24.620Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.103934524Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100582023-09-22T23:28:24.622Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.10397968Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100592023-09-22T23:28:24.625Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.1039989Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100602023-09-22T23:28:24.628Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.10404607Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100612023-09-22T23:28:24.631Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.104089865Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100622023-09-22T23:28:24.634Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.104127426Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100632023-09-22T23:28:24.638Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.104165491Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100642023-09-22T23:28:24.642Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.104224051Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
100652023-09-22T23:28:24.644ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
100662023-09-22T23:28:24.647ZLoading write count information from file "/tmp/test_repair_verify.data"
100672023-09-22T23:28:24.671ZRun Repair workload
100682023-09-22T23:28:24.674Z01/30 Write block 148 len 28672 data: 6 7 7 8 7 5 5
100692023-09-22T23:28:24.677Z02/30 Write block 328 len 8192 data: 4 3
100702023-09-22T23:28:24.680Z03/30 Read block 292 len 12288
100712023-09-22T23:28:24.683Z04/30 Read block 38 len 24576
100722023-09-22T23:28:24.686Z05/30 Read block 558 len 36864
100732023-09-22T23:28:24.689Z06/30 Read block 306 len 12288
100742023-09-22T23:28:24.692Z07/30 Write block 342 len 16384 data: 5 5 6 6
100752023-09-22T23:28:24.695Z08/30 Write block 202 len 40960 data: 7 7 6 10 11 10 12 10 11 12
100762023-09-22T23:28:24.698Z09/30 Flush
100772023-09-22T23:28:24.701Z10/30 Flush
100782023-09-22T23:28:24.703Z11/30 Write block 481 len 28672 data: 6 7 6 8 8 9 7
100792023-09-22T23:28:24.706Z12/30 Write block 416 len 24576 data: 7 7 6 7 7 7
100802023-09-22T23:28:24.709Z13/30 Write block 543 len 32768 data: 6 6 5 7 5 9 11 10
100812023-09-22T23:28:24.712Z14/30 Write block 575 len 32768 data: 5 5 6 5 4 4 4 4
100822023-09-22T23:28:24.715Z15/30 Read block 541 len 32768
100832023-09-22T23:28:24.717Z16/30 Flush
100842023-09-22T23:28:24.720Z17/30 Write block 576 len 32768 data: 6 7 6 5 5 5 5 5
100852023-09-22T23:28:24.723Z18/30 Write block 127 len 16384 data: 6 6 5 5
100862023-09-22T23:28:24.726Z19/30 Read block 60 len 24576
100872023-09-22T23:28:24.729Z20/30 Read block 250 len 20480
100882023-09-22T23:28:24.732Z21/30 Read block 92 len 32768
100892023-09-22T23:28:24.735Z22/30 Read block 533 len 8192
100902023-09-22T23:28:24.738Z23/30 Read block 521 len 32768
100912023-09-22T23:28:24.740Z24/30 Read block 467 len 8192
100922023-09-22T23:28:24.743ZError: Difference in extent metadata found!
100932023-09-22T23:28:24.746Z25/30 Read block 543 len 36864
100942023-09-22T23:28:24.749Z26/30 Read block 117 len 28672
100952023-09-22T23:28:24.752Z27/30 Read block 232 len 36864
100962023-09-22T23:28:24.755Z28/30 Write block 326 len 28672 data: 5 3 5 4 3 4 4
100972023-09-22T23:28:24.758Z29/30 Read block 264 len 4096
100982023-09-22T23:28:24.760Z30/30 Read block 154 len 36864
100992023-09-22T23:28:24.763Z----------------------------------------------------------------
101002023-09-22T23:28:24.766Z Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:30
101012023-09-22T23:28:24.769ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
101022023-09-22T23:28:24.771Z 1 Acked 1000 Write 7 Done Done Done false
101032023-09-22T23:28:25.177Z 2 Acked 1001 Write 2 Done Sent Done false
101042023-09-22T23:28:25.180Z 3 Acked 1002 Read 3 Done Sent Done false
101052023-09-22T23:28:25.183Z 4 Acked 1003 Read 6 Done Sent Done false
101062023-09-22T23:28:25.186Z 5 Acked 1004 Read 9 Done Sent Done false
101072023-09-22T23:28:25.189Z 6 Acked 1005 Read 3 Done Sent Done false
101082023-09-22T23:28:25.192Z 7 Acked 1006 Write 4 Done Sent Done false
101092023-09-22T23:28:25.195Z 8 Acked 1007 Write 10 Done Sent Done false
101102023-09-22T23:28:25.198Z 9 Acked 1008 Flush 0 Done Sent Done false
101112023-09-22T23:28:25.201Z 10 Acked 1009 Flush 0 Done Sent Done false
101122023-09-22T23:28:25.204Z 11 Acked 1010 Write 7 Done Sent Done false
101132023-09-22T23:28:25.207Z 12 Acked 1011 Write 6 Done Sent Done false
101142023-09-22T23:28:25.210Z 13 Acked 1012 Write 8 Done Sent Done false
101152023-09-22T23:28:25.218Z 14 Acked 1013 Write 8 Done Sent Done false
101162023-09-22T23:28:25.230Z 15 Acked 1014 Read 8 Done Sent Done false
101172023-09-22T23:28:25.233Z 16 Acked 1015 Flush 0 Done Sent Done false
101182023-09-22T23:28:25.236Z 17 Acked 1016 Write 8 Done Sent Done false
101192023-09-22T23:28:25.241Z 18 Acked 1017 Write 4 Done Sent Done false
101202023-09-22T23:28:25.245Z 19 Acked 1018 Read 6 Done Sent Done false
101212023-09-22T23:28:25.247Z 20 Acked 1019 Read 5 Done Sent Done false
101222023-09-22T23:28:25.250Z 21 Acked 1020 Read 8 Done Sent Done false
101232023-09-22T23:28:25.253Z 22 Acked 1021 Read 2 Done Sent Done false
101242023-09-22T23:28:25.256Z 23 Acked 1022 Read 8 Done Sent Done false
101252023-09-22T23:28:25.259Z 24 Acked 1023 Read 2 Done Sent Done false
101262023-09-22T23:28:25.262Z 25 Acked 1024 Read 9 Done Sent Done false
101272023-09-22T23:28:25.265Z 26 Acked 1025 Read 7 Done Sent Done false
101282023-09-22T23:28:25.267Z 27 Acked 1026 Read 9 Done Sent Done false
101292023-09-22T23:28:25.270Z 28 Acked 1027 Write 7 Done Sent Done false
101302023-09-22T23:28:25.273Z 29 Acked 1028 Read 1 Done Sent Done false
101312023-09-22T23:28:25.275Z 30 Acked 1029 Read 9 Done Sent Sent false
101322023-09-22T23:28:25.278Z STATES DS:0 DS:1 DS:2 TOTAL
101332023-09-22T23:28:25.281Z New 0 0 0 0
101342023-09-22T23:28:25.284Z Sent 0 29 1 30
101352023-09-22T23:28:25.287Z Done 30 1 29 60
101362023-09-22T23:28:25.290Z Skipped 0 0 0 0
101372023-09-22T23:28:25.293Z Error 0 0 0 0
101382023-09-22T23:28:25.296ZLast Flush: 1015 0 1015
101392023-09-22T23:28:25.298ZDownstairs last five completed:
101402023-09-22T23:28:25.301ZUpstairs last five completed: 30 29 28 27 26 25
101412023-09-22T23:28:25.304ZWrote out file "/tmp/test_repair_verify.data"
101422023-09-22T23:28:25.307Z{"msg":"127.0.0.1:8830: proc: channel closed","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:28:24.211008229Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469,"looper":"2"}
101432023-09-22T23:28:25.309Z{"msg":"[2] f1559b4b-06ad-47ec-9b3d-a0691a8f0cf0 Gone missing, transition from Active to Offline","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.211075758Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469}
101442023-09-22T23:28:25.312Z{"msg":"[2] f1559b4b-06ad-47ec-9b3d-a0691a8f0cf0 connection to 127.0.0.1:8830 closed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.211208323Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469,"looper":"2"}
101452023-09-22T23:28:25.315Z{"msg":"127.0.0.1:8830 Message to ds_status_tx failed: SendError { .. }","v":0,"name":"crucible","level":50,"time":"2023-09-22T23:28:24.211261512Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1469,"looper":"2"}
101462023-09-22T23:28:25.318Z
101472023-09-22T23:28:25.321ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
101482023-09-22T23:28:25.324Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.243104269Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1470}
101492023-09-22T23:28:25.326Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.243365587Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1470}
101502023-09-22T23:28:25.329Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.243388605Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1470}
101512023-09-22T23:28:25.332Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.268850724Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1470}
101522023-09-22T23:28:25.335Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.268881228Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1470}
101532023-09-22T23:28:25.338Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.268900125Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1470}
101542023-09-22T23:28:25.341Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.292248416Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1470}
101552023-09-22T23:28:25.344Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.292278859Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1470}
101562023-09-22T23:28:25.347Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:24.292297394Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1470}
101572023-09-22T23:28:25.350ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
101582023-09-22T23:28:25.353Z 0 000-019 60 60 60 90 90 90 F F F
101592023-09-22T23:28:25.356Z 1 020-039 74 74 74 111 111 111 F F F
101602023-09-22T23:28:25.359Z 2 040-059 62 62 62 92 92 92 F F F
101612023-09-22T23:28:25.362Z 3 060-079 74 74 74 111 111 111 F F F
101622023-09-22T23:28:25.364Z 4 080-099 74 74 74 111 111 111 F F F
101632023-09-22T23:28:25.367Z 5 100-119 70 70 70 103 103 103 F F F
101642023-09-22T23:28:25.370Z 6 120-139 69 69 69 100 100 100 T F T <---
101652023-09-22T23:28:25.373Z 7 140-159 76 72 76 113 107 113 F T F <---
101662023-09-22T23:28:25.376Z 8 160-179 69 69 69 100 100 100 F F F
101672023-09-22T23:28:25.378Z 9 180-199 71 71 71 104 104 104 F F F
101682023-09-22T23:28:25.381Z 10 200-219 76 75 76 113 112 113 F F F <---
101692023-09-22T23:28:25.384Z 11 220-239 72 72 72 108 108 108 F F F
101702023-09-22T23:28:25.387Z 12 240-259 73 73 73 109 109 109 F F F
101712023-09-22T23:28:25.390Z 13 260-279 74 74 74 111 111 111 F F F
101722023-09-22T23:28:25.394Z 14 280-299 75 75 75 112 112 112 F F F
101732023-09-22T23:28:25.396Z 15 300-319 69 69 69 100 100 100 F F F
101742023-09-22T23:28:25.400Z 16 320-339 76 69 76 113 100 113 T F T <---
101752023-09-22T23:28:25.403Z 17 340-359 76 66 76 113 95 113 F F F <---
101762023-09-22T23:28:25.406Z 18 360-379 71 71 71 104 104 104 F F F
101772023-09-22T23:28:25.409Z 19 380-399 71 71 71 104 104 104 F F F
101782023-09-22T23:28:25.412Z 20 400-419 76 70 76 115 101 115 F F F <---
101792023-09-22T23:28:25.414Z 21 420-439 76 70 76 115 101 115 F F F <---
101802023-09-22T23:28:25.417Z 22 440-459 75 75 75 112 112 112 F F F
101812023-09-22T23:28:25.420Z 23 460-479 71 71 71 104 104 104 F F F
101822023-09-22T23:28:25.423Z 24 480-499 76 75 76 115 112 115 F F F <---
101832023-09-22T23:28:25.426Z 25 500-519 74 74 74 110 110 110 F F F
101842023-09-22T23:28:25.429Z 26 520-539 74 74 74 111 111 111 F F F
101852023-09-22T23:28:25.434Z 27 540-559 76 71 76 115 104 115 F F F <---
101862023-09-22T23:28:25.438Z 28 560-579 76 74 76 115 111 115 T F T <---
101872023-09-22T23:28:25.441Z 29 580-599 76 72 76 115 105 115 T F T <---
101882023-09-22T23:28:25.444ZMax gen: 76, Max flush: 115
101892023-09-22T23:28:25.446ZA Difference in extent metadata is expected here
101902023-09-22T23:28:25.450ZOn loop 38
101912023-09-22T23:28:25.452Z
101922023-09-22T23:28:25.455ZVerifying data now
101932023-09-22T23:28:30.563ZNo change after verify
101942023-09-22T23:28:30.567ZLoop: 38 Downstairs dump after verify (and repair):
101952023-09-22T23:28:30.586Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.581461619Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1475}
101962023-09-22T23:28:30.593Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.581868025Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1475}
101972023-09-22T23:28:30.599Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.581892246Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1475}
101982023-09-22T23:28:30.607Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.60074769Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1475}
101992023-09-22T23:28:31.017Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.600799973Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1475}
102002023-09-22T23:28:31.021Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.600838268Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1475}
102012023-09-22T23:28:31.032Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.61188434Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1475}
102022023-09-22T23:28:31.446Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.61193158Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1475}
102032023-09-22T23:28:31.453Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.61196884Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1475}
102042023-09-22T23:28:31.465ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
102052023-09-22T23:28:31.470Z 0 000-019 60 60 60 90 90 90 F F F
102062023-09-22T23:28:31.473Z 1 020-039 74 74 74 111 111 111 F F F
102072023-09-22T23:28:31.476Z 2 040-059 62 62 62 92 92 92 F F F
102082023-09-22T23:28:31.479Z 3 060-079 74 74 74 111 111 111 F F F
102092023-09-22T23:28:31.482Z 4 080-099 74 74 74 111 111 111 F F F
102102023-09-22T23:28:31.486Z 5 100-119 70 70 70 103 103 103 F F F
102112023-09-22T23:28:31.490Z 6 120-139 77 77 77 116 116 116 F F F
102122023-09-22T23:28:31.494Z 7 140-159 76 76 76 113 113 113 F F F
102132023-09-22T23:28:31.499Z 8 160-179 69 69 69 100 100 100 F F F
102142023-09-22T23:28:31.503Z 9 180-199 71 71 71 104 104 104 F F F
102152023-09-22T23:28:31.511Z 10 200-219 76 76 76 113 113 113 F F F
102162023-09-22T23:28:31.517Z 11 220-239 72 72 72 108 108 108 F F F
102172023-09-22T23:28:31.522Z 12 240-259 73 73 73 109 109 109 F F F
102182023-09-22T23:28:31.525Z 13 260-279 74 74 74 111 111 111 F F F
102192023-09-22T23:28:31.937Z 14 280-299 75 75 75 112 112 112 F F F
102202023-09-22T23:28:31.947Z 15 300-319 69 69 69 100 100 100 F F F
102212023-09-22T23:28:31.952Z 16 320-339 77 77 77 116 116 116 F F F
102222023-09-22T23:28:31.955Z 17 340-359 76 76 76 113 113 113 F F F
102232023-09-22T23:28:31.959Z 18 360-379 71 71 71 104 104 104 F F F
102242023-09-22T23:28:31.962Z 19 380-399 71 71 71 104 104 104 F F F
102252023-09-22T23:28:31.967Z 20 400-419 76 76 76 115 115 115 F F F
102262023-09-22T23:28:31.971Z 21 420-439 76 76 76 115 115 115 F F F
102272023-09-22T23:28:31.975Z 22 440-459 75 75 75 112 112 112 F F F
102282023-09-22T23:28:31.988Z 23 460-479 71 71 71 104 104 104 F F F
102292023-09-22T23:28:31.998Z 24 480-499 76 76 76 115 115 115 F F F
102302023-09-22T23:28:32.411Z 25 500-519 74 74 74 110 110 110 F F F
102312023-09-22T23:28:32.426Z 26 520-539 74 74 74 111 111 111 F F F
102322023-09-22T23:28:32.430Z 27 540-559 76 76 76 115 115 115 F F F
102332023-09-22T23:28:32.434Z 28 560-579 77 77 77 116 116 116 F F F
102342023-09-22T23:28:32.437Z 29 580-599 77 77 77 116 116 116 F F F
102352023-09-22T23:28:32.441ZMax gen: 77, Max flush: 116
102362023-09-22T23:28:32.445Z
102372023-09-22T23:28:32.448ZBegin loop 39
102382023-09-22T23:28:32.451ZDownstairs to restart: 1
102392023-09-22T23:28:32.454Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.659225008Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102402023-09-22T23:28:32.457Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.659616233Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102412023-09-22T23:28:32.460Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.659689593Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102422023-09-22T23:28:32.463Z{"msg":"Crucible stats registered with UUID: 03386e2d-7771-46a9-b202-10f9d323d6d7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.659758654Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102432023-09-22T23:28:32.466Z{"msg":"Crucible 03386e2d-7771-46a9-b202-10f9d323d6d7 has session id: 507d0211-21ca-4514-8413-a75dd6d2932c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.659852375Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102442023-09-22T23:28:32.468ZCrucible runtime is spawned
102452023-09-22T23:28:32.472Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30The guest has requested activation with gen:78
102462023-09-22T23:28:32.475Z,"time":"2023-09-22T23:28:30.659987635Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477,"task":"up_listen"}
102472023-09-22T23:28:32.478Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.660108017Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102482023-09-22T23:28:32.481Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.660194278Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102492023-09-22T23:28:32.484Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.66036283Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477,"looper":"2"}
102502023-09-22T23:28:32.487Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.660418302Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477,"looper":"0"}
102512023-09-22T23:28:32.490Z{"msg":"[2] 03386e2d-7771-46a9-b202-10f9d323d6d7 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.660488078Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477,"looper":"2"}
102522023-09-22T23:28:32.492Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.660567666Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477,"looper":"1"}
102532023-09-22T23:28:32.495Z{"msg":"[0] 03386e2d-7771-46a9-b202-10f9d323d6d7 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.66061192Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477,"looper":"0"}
102542023-09-22T23:28:32.498Z{"msg":"03386e2d-7771-46a9-b202-10f9d323d6d7 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.660930603Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102552023-09-22T23:28:32.505Z{"msg":"Set desired generation to :78","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.660995774Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102562023-09-22T23:28:32.508Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.661083939Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102572023-09-22T23:28:32.512Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.661394242Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102582023-09-22T23:28:32.515Z{"msg":"[2] 03386e2d-7771-46a9-b202-10f9d323d6d7 (19f78d33-8720-4aac-9985-2d04b1c0885c) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.661753095Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102592023-09-22T23:28:32.519Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.66182703Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102602023-09-22T23:28:32.523Z{"msg":"[2] client is_active_req TRUE, promote! session 19f78d33-8720-4aac-9985-2d04b1c0885c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.661901345Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102612023-09-22T23:28:32.526Z{"msg":"[0] 03386e2d-7771-46a9-b202-10f9d323d6d7 (19f78d33-8720-4aac-9985-2d04b1c0885c) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.661986358Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102622023-09-22T23:28:32.531Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.662057955Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102632023-09-22T23:28:32.534Z{"msg":"[0] client is_active_req TRUE, promote! session 19f78d33-8720-4aac-9985-2d04b1c0885c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.66210585Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102642023-09-22T23:28:32.538Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.66240967Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102652023-09-22T23:28:32.541Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.662527266Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102662023-09-22T23:28:32.544Z{"msg":"03386e2d-7771-46a9-b202-10f9d323d6d7 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.662578583Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102672023-09-22T23:28:32.548Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.662617137Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102682023-09-22T23:28:32.551Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.662655708Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102692023-09-22T23:28:32.554Z{"msg":"03386e2d-7771-46a9-b202-10f9d323d6d7 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.662689166Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102702023-09-22T23:28:32.557Z{"msg":"[2] 03386e2d-7771-46a9-b202-10f9d323d6d7 (19f78d33-8720-4aac-9985-2d04b1c0885c) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.66458813Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102712023-09-22T23:28:32.559Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.664633535Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102722023-09-22T23:28:32.562Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:28:30.664673662Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102732023-09-22T23:28:32.565Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.664834872Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102742023-09-22T23:28:32.568Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.664951581Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102752023-09-22T23:28:32.571Z{"msg":"03386e2d-7771-46a9-b202-10f9d323d6d7 WaitActive New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.66497779Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102762023-09-22T23:28:32.574Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.665011725Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102772023-09-22T23:28:32.577Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.665030302Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102782023-09-22T23:28:32.579Z{"msg":"[0] 03386e2d-7771-46a9-b202-10f9d323d6d7 (19f78d33-8720-4aac-9985-2d04b1c0885c) WaitActive New WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.665246948Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102792023-09-22T23:28:32.582Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.665272479Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102802023-09-22T23:28:32.585Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:28:30.665291402Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102812023-09-22T23:28:32.588Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.665308944Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102822023-09-22T23:28:32.591Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.665327176Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102832023-09-22T23:28:32.594Z{"msg":"03386e2d-7771-46a9-b202-10f9d323d6d7 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.665345799Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102842023-09-22T23:28:32.597Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.665363559Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102852023-09-22T23:28:32.600Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:30.665380856Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102862023-09-22T23:28:32.603Z{"msg":"[1] 03386e2d-7771-46a9-b202-10f9d323d6d7 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.661273294Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477,"looper":"1"}
102872023-09-22T23:28:32.606ZError: Difference in extent metadata found!
102882023-09-22T23:28:32.609Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.661332371Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102892023-09-22T23:28:32.612Z{"msg":"[1] 03386e2d-7771-46a9-b202-10f9d323d6d7 (19f78d33-8720-4aac-9985-2d04b1c0885c) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.662487143Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102902023-09-22T23:28:32.617Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.662514005Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102912023-09-22T23:28:32.619Z{"msg":"[1] client is_active_req TRUE, promote! session 19f78d33-8720-4aac-9985-2d04b1c0885c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.662534501Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102922023-09-22T23:28:32.622Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.663148051Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102932023-09-22T23:28:32.625Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.663178678Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102942023-09-22T23:28:32.628Z{"msg":"03386e2d-7771-46a9-b202-10f9d323d6d7 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.66320782Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102952023-09-22T23:28:32.631Z{"msg":"[1] 03386e2d-7771-46a9-b202-10f9d323d6d7 (19f78d33-8720-4aac-9985-2d04b1c0885c) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.665597568Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102962023-09-22T23:28:32.634Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.665623325Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102972023-09-22T23:28:32.636Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:28:31.665642Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102982023-09-22T23:28:32.639Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.665660211Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
102992023-09-22T23:28:32.642Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.665684567Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103002023-09-22T23:28:32.645Z{"msg":"03386e2d-7771-46a9-b202-10f9d323d6d7 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.66571762Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103012023-09-22T23:28:32.648Z{"msg":"[0]R flush_numbers[0..12]: [90, 111, 92, 111, 111, 103, 116, 113, 100, 104, 113, 108]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.66574495Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103022023-09-22T23:28:32.651Z{"msg":"[0]R generation[0..12]: [60, 74, 62, 74, 74, 70, 77, 76, 69, 71, 76, 72]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.665764367Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103032023-09-22T23:28:32.653Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.665784454Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103042023-09-22T23:28:32.657Z{"msg":"[1]R flush_numbers[0..12]: [90, 111, 92, 111, 111, 103, 116, 113, 100, 104, 113, 108]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.665803808Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103052023-09-22T23:28:32.659Z{"msg":"[1]R generation[0..12]: [60, 74, 62, 74, 74, 70, 77, 76, 69, 71, 76, 72]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.66582258Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103062023-09-22T23:28:32.662Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.66584203Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103072023-09-22T23:28:32.665Z{"msg":"[2]R flush_numbers[0..12]: [90, 111, 92, 111, 111, 103, 116, 113, 100, 104, 113, 108]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.665861087Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103082023-09-22T23:28:32.668Z{"msg":"[2]R generation[0..12]: [60, 74, 62, 74, 74, 70, 77, 76, 69, 71, 76, 72]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.665879516Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103092023-09-22T23:28:32.671Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.66589915Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103102023-09-22T23:28:32.675Z{"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.665916642Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103112023-09-22T23:28:32.678Z{"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.66593451Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103122023-09-22T23:28:32.681Z{"msg":"Next flush: 117","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.665951908Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103132023-09-22T23:28:32.684Z{"msg":"All extents match","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:78
103142023-09-22T23:28:32.687Z,"time":"2023-09-22T23:28:31.665968849Z","hostname":"Wait for a query_work_queue command to finish before sending IO
103152023-09-22T23:28:32.691Zip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103162023-09-22T23:28:32.695Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.665989491Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103172023-09-22T23:28:32.698Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.666007534Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103182023-09-22T23:28:32.701Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.66602515Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103192023-09-22T23:28:32.704Z{"msg":"03386e2d-7771-46a9-b202-10f9d323d6d7 is now active with session: 19f78d33-8720-4aac-9985-2d04b1c0885c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.666044413Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103202023-09-22T23:28:32.707Z{"msg":"03386e2d-7771-46a9-b202-10f9d323d6d7 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.666063036Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103212023-09-22T23:28:32.710Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.666080817Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103222023-09-22T23:28:32.713Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.666104239Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103232023-09-22T23:28:32.716Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.666135396Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103242023-09-22T23:28:32.719Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.666170006Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103252023-09-22T23:28:32.721Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.666209302Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103262023-09-22T23:28:32.724Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.666243683Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103272023-09-22T23:28:32.727Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.666276216Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103282023-09-22T23:28:32.730Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.666307323Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103292023-09-22T23:28:32.732Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.666341492Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103302023-09-22T23:28:32.735Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.666371966Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103312023-09-22T23:28:32.738Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.66640614Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1477}
103322023-09-22T23:28:32.741ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
103332023-09-22T23:28:32.744ZLoading write count information from file "/tmp/test_repair_verify.data"
103342023-09-22T23:28:32.747ZRun Repair workload
103352023-09-22T23:28:32.750Z01/30 Write block 392 len 28672 data: 5 4 5 6 6 6 7
103362023-09-22T23:28:32.754Z02/30 Read block 187 len 24576
103372023-09-22T23:28:32.757Z03/30 Flush
103382023-09-22T23:28:32.760Z04/30 Write block 476 len 20480 data: 6 6 7 8 7
103392023-09-22T23:28:32.763Z05/30 Read block 102 len 4096
103402023-09-22T23:28:32.768Z06/30 Write block 509 len 28672 data: 7 6 5 8 10 11 10
103412023-09-22T23:28:32.771Z07/30 Read block 385 len 24576
103422023-09-22T23:28:32.775Z08/30 Read block 282 len 36864
103432023-09-22T23:28:32.778Z09/30 Read block 229 len 28672
103442023-09-22T23:28:32.781Z10/30 Write block 170 len 32768 data: 7 8 9 9 9 10 10 9
103452023-09-22T23:28:32.784Z11/30 Flush
103462023-09-22T23:28:32.787Z12/30 Write block 495 len 20480 data: 10 11 11 7 8
103472023-09-22T23:28:32.790Z13/30 Read block 179 len 24576
103482023-09-22T23:28:32.793Z14/30 Write block 482 len 28672 data: 8 7 9 9 10 8 10
103492023-09-22T23:28:32.796Z15/30 Write block 514 len 36864 data: 12 11 11 11 11 9 8 6 7
103502023-09-22T23:28:32.799Z16/30 Flush
103512023-09-22T23:28:32.802Z17/30 Flush
103522023-09-22T23:28:32.805Z18/30 Write block 426 len 12288 data: 4 6 6
103532023-09-22T23:28:32.808Z19/30 Write block 227 len 24576 data: 6 5 5 7 3 3
103542023-09-22T23:28:32.811Z20/30 Write block 191 len 12288 data: 6 5 5
103552023-09-22T23:28:32.814Z21/30 Write block 359 len 40960 data: 6 7 6 5 5 4 4 5 6 7
103562023-09-22T23:28:32.818Z22/30 Flush
103572023-09-22T23:28:32.821Z23/30 Write block 464 len 20480 data: 7 10 8 8 8
103582023-09-22T23:28:32.847Z24/30 Read block 92 len 12288
103592023-09-22T23:28:32.851Z25/30 Read block 235 len 28672
103602023-09-22T23:28:32.854Z26/30 Read block 486 len 40960
103612023-09-22T23:28:32.857Z27/30 Read block 301 len 24576
103622023-09-22T23:28:32.861Z28/30 Write block 210 len 32768 data: 12 13 9 10 8 8 7 7
103632023-09-22T23:28:32.865Z29/30 Write block 102 len 16384 data: 6 7 9 11
103642023-09-22T23:28:32.868Z30/30 Write block 429 len 8192 data: 8 8
103652023-09-22T23:28:32.871Z----------------------------------------------------------------
103662023-09-22T23:28:32.874Z Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30
103672023-09-22T23:28:32.876ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
103682023-09-22T23:28:32.879Z 1 Acked 1000 Write 7 Done Done Done false
103692023-09-22T23:28:32.882Z 2 Acked 1001 Read 6 Done Sent Done false
103702023-09-22T23:28:32.885Z 3 Acked 1002 Flush 0 Done Sent Done false
103712023-09-22T23:28:32.888Z 4 Acked 1003 Write 5 Done Sent Done false
103722023-09-22T23:28:32.891Z 5 Acked 1004 Read 1 Done Sent Done false
103732023-09-22T23:28:32.894Z 6 Acked 1005 Write 7 Done Sent Done false
103742023-09-22T23:28:32.898Z 7 Acked 1006 Read 6 Done Sent Done false
103752023-09-22T23:28:32.902Z 8 Acked 1007 Read 9 Done Sent Done false
103762023-09-22T23:28:32.905Z 9 Acked 1008 Read 7 Done Sent Done false
103772023-09-22T23:28:32.909Z 10 Acked 1009 Write 8 Done Sent Done false
103782023-09-22T23:28:32.911Z 11 Acked 1010 Flush 0 Done Sent Done false
103792023-09-22T23:28:32.914Z 12 Acked 1011 Write 5 Done Sent Done false
103802023-09-22T23:28:32.917Z 13 Acked 1012 Read 6 Done Sent Done false
103812023-09-22T23:28:32.920Z 14 Acked 1013 Write 7 Done Sent Done false
103822023-09-22T23:28:32.923Z 15 Acked 1014 Write 9 Done Sent Done false
103832023-09-22T23:28:32.927Z 16 Acked 1015 Flush 0 Done Sent Done false
103842023-09-22T23:28:32.931Z 17 Acked 1016 Flush 0 Done Sent Done false
103852023-09-22T23:28:32.936Z 18 Acked 1017 Write 3 Done Sent Done false
103862023-09-22T23:28:32.940Z 19 Acked 1018 Write 6 Done Sent Done false
103872023-09-22T23:28:32.944Z 20 Acked 1019 Write 3 Done Sent Done false
103882023-09-22T23:28:32.947Z 21 Acked 1020 Write 10 Done Sent Done false
103892023-09-22T23:28:32.950Z 22 Acked 1021 Flush 0 Done Sent Done false
103902023-09-22T23:28:32.954Z 23 Acked 1022 Write 5 Done Sent Done false
103912023-09-22T23:28:32.957Z 24 Acked 1023 Read 3 Done Sent Done false
103922023-09-22T23:28:32.960Z 25 Acked 1024 Read 7 Done Sent Done false
103932023-09-22T23:28:32.963Z 26 Acked 1025 Read 10 Done Sent Done false
103942023-09-22T23:28:32.966Z 27 Acked 1026 Read 6 Done Sent Done false
103952023-09-22T23:28:32.969Z 28 Acked 1027 Write 8 Sent Sent Sent false
103962023-09-22T23:28:32.972Z 29 Acked 1028 Write 4 Sent Sent Sent false
103972023-09-22T23:28:32.977Z 30 Acked 1029 Write 2 Sent Sent Sent false
103982023-09-22T23:28:32.980Z STATES DS:0 DS:1 DS:2 TOTAL
103992023-09-22T23:28:32.984Z New 0 0 0 0
104002023-09-22T23:28:32.988Z Sent 3 29 3 35
104012023-09-22T23:28:32.992Z Done 27 1 27 55
104022023-09-22T23:28:32.996Z Skipped 0 0 0 0
104032023-09-22T23:28:32.999Z Error 0 0 0 0
104042023-09-22T23:28:33.003ZLast Flush: 1021 0 1021
104052023-09-22T23:28:33.006ZDownstairs last five completed:
104062023-09-22T23:28:33.009ZUpstairs last five completed: 30 29 28 27 26 25
104072023-09-22T23:28:33.012ZWrote out file "/tmp/test_repair_verify.data"
104082023-09-22T23:28:33.017Z
104092023-09-22T23:28:33.020ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
104102023-09-22T23:28:33.023Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.801600143Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1478}
104112023-09-22T23:28:33.026Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.801851667Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1478}
104122023-09-22T23:28:33.031Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.801874224Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1478}
104132023-09-22T23:28:33.035Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.82542368Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1478}
104142023-09-22T23:28:33.039Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.8254596Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1478}
104152023-09-22T23:28:33.042Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.825479865Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1478}
104162023-09-22T23:28:33.047Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.844440204Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1478}
104172023-09-22T23:28:33.051Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.844468921Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1478}
104182023-09-22T23:28:33.054Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:31.844487258Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1478}
104192023-09-22T23:28:33.058ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
104202023-09-22T23:28:33.061Z 0 000-019 60 60 60 90 90 90 F F F
104212023-09-22T23:28:33.064Z 1 020-039 74 74 74 111 111 111 F F F
104222023-09-22T23:28:33.067Z 2 040-059 62 62 62 92 92 92 F F F
104232023-09-22T23:28:33.070Z 3 060-079 74 74 74 111 111 111 F F F
104242023-09-22T23:28:33.073Z 4 080-099 74 74 74 111 111 111 F F F
104252023-09-22T23:28:33.076Z 5 100-119 70 70 70 103 103 103 T F T <---
104262023-09-22T23:28:33.079Z 6 120-139 77 77 77 116 116 116 F F F
104272023-09-22T23:28:33.083Z 7 140-159 76 76 76 113 113 113 F F F
104282023-09-22T23:28:33.086Z 8 160-179 78 69 78 118 100 118 F F F <---
104292023-09-22T23:28:33.090Z 9 180-199 78 71 78 121 104 121 F F F <---
104302023-09-22T23:28:33.093Z 10 200-219 76 76 76 113 113 113 T F T <---
104312023-09-22T23:28:33.097Z 11 220-239 78 72 78 121 108 121 F F F <---
104322023-09-22T23:28:33.100Z 12 240-259 73 73 73 109 109 109 F F F
104332023-09-22T23:28:33.104Z 13 260-279 74 74 74 111 111 111 F F F
104342023-09-22T23:28:33.107Z 14 280-299 75 75 75 112 112 112 F F F
104352023-09-22T23:28:33.111Z 15 300-319 69 69 69 100 100 100 F F F
104362023-09-22T23:28:33.113Z 16 320-339 77 77 77 116 116 116 F F F
104372023-09-22T23:28:33.116Z 17 340-359 78 76 78 121 113 121 F F F <---
104382023-09-22T23:28:33.119Z 18 360-379 78 71 78 121 104 121 F F F <---
104392023-09-22T23:28:33.122Z 19 380-399 78 71 78 117 104 117 F T F <---
104402023-09-22T23:28:33.125Z 20 400-419 76 76 76 115 115 115 F F F
104412023-09-22T23:28:33.129Z 21 420-439 78 76 78 121 115 121 F F F <---
104422023-09-22T23:28:33.132Z 22 440-459 75 75 75 112 112 112 F F F
104432023-09-22T23:28:33.137Z 23 460-479 78 71 78 118 104 118 T F T <---
104442023-09-22T23:28:33.141Z 24 480-499 78 76 78 119 115 119 F F F <---
104452023-09-22T23:28:33.145Z 25 500-519 78 74 78 119 110 119 F F F <---
104462023-09-22T23:28:33.150Z 26 520-539 78 74 78 119 111 119 F F F <---
104472023-09-22T23:28:33.154Z 27 540-559 76 76 76 115 115 115 F F F
104482023-09-22T23:28:33.156Z 28 560-579 77 77 77 116 116 116 F F F
104492023-09-22T23:28:33.159Z 29 580-599 77 77 77 116 116 116 F F F
104502023-09-22T23:28:33.162ZMax gen: 78, Max flush: 121
104512023-09-22T23:28:33.165ZA Difference in extent metadata is expected here
104522023-09-22T23:28:33.168ZOn loop 39
104532023-09-22T23:28:33.171Z
104542023-09-22T23:28:33.174ZVerifying data now
104552023-09-22T23:28:38.372ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
104562023-09-22T23:28:38.378Z432,433c432,433
104572023-09-22T23:28:38.382Z< 7,
104582023-09-22T23:28:38.386Z< 7,
104592023-09-22T23:28:38.390Z---
104602023-09-22T23:28:38.394Z> 8,
104612023-09-22T23:28:38.400Z> 8,
104622023-09-22T23:28:38.404Zdiff found after verify
104632023-09-22T23:28:38.409ZLoop: 39 Downstairs dump after verify (and repair):
104642023-09-22T23:28:38.413Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.399043592Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1486}
104652023-09-22T23:28:38.417Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.399390509Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1486}
104662023-09-22T23:28:38.421Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.399434424Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1486}
104672023-09-22T23:28:38.425Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.418168343Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1486}
104682023-09-22T23:28:38.429Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.418220726Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1486}
104692023-09-22T23:28:38.434Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.418258832Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1486}
104702023-09-22T23:28:38.438Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.429352577Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1486}
104712023-09-22T23:28:38.442Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.429403626Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1486}
104722023-09-22T23:28:38.447Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.429441316Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1486}
104732023-09-22T23:28:38.450ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
104742023-09-22T23:28:38.455Z 0 000-019 60 60 60 90 90 90 F F F
104752023-09-22T23:28:38.459Z 1 020-039 74 74 74 111 111 111 F F F
104762023-09-22T23:28:38.463Z 2 040-059 62 62 62 92 92 92 F F F
104772023-09-22T23:28:38.467Z 3 060-079 74 74 74 111 111 111 F F F
104782023-09-22T23:28:38.472Z 4 080-099 74 74 74 111 111 111 F F F
104792023-09-22T23:28:38.476Z 5 100-119 79 79 79 122 122 122 F F F
104802023-09-22T23:28:38.480Z 6 120-139 77 77 77 116 116 116 F F F
104812023-09-22T23:28:38.484Z 7 140-159 76 76 76 113 113 113 F F F
104822023-09-22T23:28:38.488Z 8 160-179 78 78 78 118 118 118 F F F
104832023-09-22T23:28:38.495Z 9 180-199 78 78 78 121 121 121 F F F
104842023-09-22T23:28:38.499Z 10 200-219 79 79 79 122 122 122 F F F
104852023-09-22T23:28:38.503Z 11 220-239 78 78 78 121 121 121 F F F
104862023-09-22T23:28:38.507Z 12 240-259 73 73 73 109 109 109 F F F
104872023-09-22T23:28:38.512Z 13 260-279 74 74 74 111 111 111 F F F
104882023-09-22T23:28:38.516Z 14 280-299 75 75 75 112 112 112 F F F
104892023-09-22T23:28:38.520Z 15 300-319 69 69 69 100 100 100 F F F
104902023-09-22T23:28:38.524Z 16 320-339 77 77 77 116 116 116 F F F
104912023-09-22T23:28:38.529Z 17 340-359 78 78 78 121 121 121 F F F
104922023-09-22T23:28:38.533Z 18 360-379 78 78 78 121 121 121 F F F
104932023-09-22T23:28:38.537Z 19 380-399 78 78 78 117 117 117 F F F
104942023-09-22T23:28:38.540Z 20 400-419 76 76 76 115 115 115 F F F
104952023-09-22T23:28:38.544Z 21 420-439 78 78 78 121 121 121 F F F
104962023-09-22T23:28:38.549Z 22 440-459 75 75 75 112 112 112 F F F
104972023-09-22T23:28:38.560Z 23 460-479 79 79 79 122 122 122 F F F
104982023-09-22T23:28:38.564Z 24 480-499 78 78 78 119 119 119 F F F
104992023-09-22T23:28:38.568Z 25 500-519 78 78 78 119 119 119 F F F
105002023-09-22T23:28:38.572Z 26 520-539 78 78 78 119 119 119 F F F
105012023-09-22T23:28:38.576Z 27 540-559 76 76 76 115 115 115 F F F
105022023-09-22T23:28:38.579Z 28 560-579 77 77 77 116 116 116 F F F
105032023-09-22T23:28:38.583Z 29 580-599 77 77 77 116 116 116 F F F
105042023-09-22T23:28:38.587ZMax gen: 79, Max flush: 122
105052023-09-22T23:28:38.595Z
105062023-09-22T23:28:38.603ZBegin loop 40
105072023-09-22T23:28:38.607ZDownstairs to restart: 2
105082023-09-22T23:28:38.614Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.477932905Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105092023-09-22T23:28:38.618Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.47830194Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105102023-09-22T23:28:38.622Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.478367677Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105112023-09-22T23:28:38.626Z{"msg":"Crucible stats registered with UUID: d66f4b4b-419e-4d97-9035-b50546b4b09a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.478399648Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105122023-09-22T23:28:38.629Z{"msg":"Crucible d66f4b4b-419e-4d97-9035-b50546b4b09a has session id: 20a338ad-3628-471e-b860-0082ccebd98d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.478464404Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105132023-09-22T23:28:38.633ZCrucible runtime is spawned
105142023-09-22T23:28:38.636ZThe guest has requested activation with gen:80
105152023-09-22T23:28:38.640Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.478896656Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488,"task":"up_listen"}
105162023-09-22T23:28:38.643Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.479011377Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105172023-09-22T23:28:38.647Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.479079065Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105182023-09-22T23:28:38.651Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.479147466Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488,"looper":"2"}
105192023-09-22T23:28:38.654Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.479266854Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488,"looper":"0"}
105202023-09-22T23:28:38.658Z{"msg":"[0] d66f4b4b-419e-4d97-9035-b50546b4b09a looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.479503829Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488,"looper":"0"}
105212023-09-22T23:28:38.664Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.479570218Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488,"looper":"1"}
105222023-09-22T23:28:38.669Z{"msg":"[1] d66f4b4b-419e-4d97-9035-b50546b4b09a looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.479772668Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488,"looper":"1"}
105232023-09-22T23:28:38.672Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.479839785Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105242023-09-22T23:28:38.676Z{"msg":"d66f4b4b-419e-4d97-9035-b50546b4b09a active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.479884229Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105252023-09-22T23:28:38.679Z{"msg":"Set desired generation to :80","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.47990663Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105262023-09-22T23:28:38.682Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.480338366Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105272023-09-22T23:28:38.686Z{"msg":"[0] d66f4b4b-419e-4d97-9035-b50546b4b09a (db6893c6-3fad-49bd-8ad6-b8f6806f8429) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.480577485Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105282023-09-22T23:28:38.690Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.480684044Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105292023-09-22T23:28:38.693Z{"msg":"[0] client is_active_req TRUE, promote! session db6893c6-3fad-49bd-8ad6-b8f6806f8429","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.480758374Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105302023-09-22T23:28:38.697Z{"msg":"[1] d66f4b4b-419e-4d97-9035-b50546b4b09a (db6893c6-3fad-49bd-8ad6-b8f6806f8429) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.480820618Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105312023-09-22T23:28:38.700Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.480877998Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105322023-09-22T23:28:38.703Z{"msg":"[1] client is_active_req TRUE, promote! session db6893c6-3fad-49bd-8ad6-b8f6806f8429","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.480916999Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105332023-09-22T23:28:38.706Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.481354508Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105342023-09-22T23:28:38.709Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.481421162Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105352023-09-22T23:28:38.712Z{"msg":"d66f4b4b-419e-4d97-9035-b50546b4b09a WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.481448055Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105362023-09-22T23:28:38.715Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.481565172Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105372023-09-22T23:28:38.718Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.481638961Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105382023-09-22T23:28:38.721Z{"msg":"d66f4b4b-419e-4d97-9035-b50546b4b09a WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.481714Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105392023-09-22T23:28:38.725Z{"msg":"[1] d66f4b4b-419e-4d97-9035-b50546b4b09a (db6893c6-3fad-49bd-8ad6-b8f6806f8429) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.483577264Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105402023-09-22T23:28:38.729Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.483635271Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105412023-09-22T23:28:38.732Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:28:38.483693696Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105422023-09-22T23:28:38.736Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.483766112Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105432023-09-22T23:28:38.739Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.483820957Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105442023-09-22T23:28:38.742Z{"msg":"d66f4b4b-419e-4d97-9035-b50546b4b09a WaitActive WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.483862776Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105452023-09-22T23:28:38.746Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.483902383Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105462023-09-22T23:28:38.749Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.483940455Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105472023-09-22T23:28:38.752Z{"msg":"[0] d66f4b4b-419e-4d97-9035-b50546b4b09a (db6893c6-3fad-49bd-8ad6-b8f6806f8429) WaitActive WaitQuorum New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.483981638Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105482023-09-22T23:28:38.755Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.48400227Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105492023-09-22T23:28:38.758Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:28:38.484020202Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105502023-09-22T23:28:38.762Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.484037168Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105512023-09-22T23:28:38.765Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.484054725Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105522023-09-22T23:28:38.769Z{"msg":"d66f4b4b-419e-4d97-9035-b50546b4b09a WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.484072661Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105532023-09-22T23:28:38.773Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.484090302Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105542023-09-22T23:28:38.776Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:38.484107123Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105552023-09-22T23:28:39.481Z{"msg":"[2] d66f4b4b-419e-4d97-9035-b50546b4b09a looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.481064682Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488,"looper":"2"}
105562023-09-22T23:28:39.485Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.481150528Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105572023-09-22T23:28:39.489Z{"msg":"[2] d66f4b4b-419e-4d97-9035-b50546b4b09a (db6893c6-3fad-49bd-8ad6-b8f6806f8429) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.482237466Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105582023-09-22T23:28:39.492Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.482298133Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105592023-09-22T23:28:39.495Z{"msg":"[2] client is_active_req TRUE, promote! session db6893c6-3fad-49bd-8ad6-b8f6806f8429","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.482341128Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105602023-09-22T23:28:39.499Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.482938355Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105612023-09-22T23:28:39.503Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.482993612Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105622023-09-22T23:28:39.506Z{"msg":"d66f4b4b-419e-4d97-9035-b50546b4b09a WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.483035825Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105632023-09-22T23:28:39.512Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.484190261Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105642023-09-22T23:28:39.516Z{"msg":"[2] d66f4b4b-419e-4d97-9035-b50546b4b09a (db6893c6-3fad-49bd-8ad6-b8f6806f8429) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.485473916Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105652023-09-22T23:28:39.522Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.485529282Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105662023-09-22T23:28:39.525Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:28:39.485552632Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105672023-09-22T23:28:39.528Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.485594629Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105682023-09-22T23:28:39.531Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.485616554Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105692023-09-22T23:28:39.535Z{"msg":"d66f4b4b-419e-4d97-9035-b50546b4b09a WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.485653837Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105702023-09-22T23:28:39.538Z{"msg":"[0]R flush_numbers[0..12]: [90, 111, 92, 111, 111, 122, 116, 113, 118, 121, 122, 121]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.485693914Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105712023-09-22T23:28:39.541Z{"msg":"[0]R generation[0..12]: [60, 74, 62, 74, 74, 79, 77, 76, 78, 78, 79, 78]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.485734287Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105722023-09-22T23:28:39.544Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.485775362Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105732023-09-22T23:28:39.550Z{"msg":"[1]R flush_numbers[0..12]: [90, 111, 92, 111, 111, 122, 116, 113, 118, 121, 122, 121]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.485815527Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105742023-09-22T23:28:39.554Z{"msg":"[1]R generation[0..12]: [60, 74, 62, 74, 74, 79, 77, 76, 78, 78, 79, 78]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:80
105752023-09-22T23:28:39.557ZWait for a query_work_queue command to finish before sending IO
105762023-09-22T23:28:39.564Z,"time":"2023-09-22T23:28:39.485880524Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105772023-09-22T23:28:39.569Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.485969805Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105782023-09-22T23:28:39.572Z{"msg":"[2]R flush_numbers[0..12]: [90, 111, 92, 111, 111, 122, 116, 113, 118, 121, 122, 121]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.486039561Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105792023-09-22T23:28:39.577Z{"msg":"[2]R generation[0..12]: [60, 74, 62, 74, 74, 79, 77, 76, 78, 78, 79, 78]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.486110064Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105802023-09-22T23:28:39.582Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.486182125Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105812023-09-22T23:28:39.585Z{"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.486217472Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105822023-09-22T23:28:39.591Z{"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.486251597Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105832023-09-22T23:28:39.600Z{"msg":"Next flush: 123","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.486286795Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105842023-09-22T23:28:39.611Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.486318077Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105852023-09-22T23:28:39.614Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.486421195Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105862023-09-22T23:28:39.617Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.486470662Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105872023-09-22T23:28:39.621Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.486509822Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105882023-09-22T23:28:39.625Z{"msg":"d66f4b4b-419e-4d97-9035-b50546b4b09a is now active with session: db6893c6-3fad-49bd-8ad6-b8f6806f8429","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.486555102Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105892023-09-22T23:28:39.627Z{"msg":"d66f4b4b-419e-4d97-9035-b50546b4b09a Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.486602631Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105902023-09-22T23:28:39.631Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.486642926Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105912023-09-22T23:28:39.634Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.486681477Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105922023-09-22T23:28:39.637Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.486727188Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105932023-09-22T23:28:39.640Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.486770869Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105942023-09-22T23:28:39.643Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.486809111Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105952023-09-22T23:28:39.647Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.486848162Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105962023-09-22T23:28:39.650Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.486885955Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105972023-09-22T23:28:39.653Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
105982023-09-22T23:28:39.657Z,"time":"2023-09-22T23:28:39.48692639Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
105992023-09-22T23:28:39.660Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.487025659Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
106002023-09-22T23:28:39.663Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.487071193Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
106012023-09-22T23:28:39.666Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.487109456Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1488}
106022023-09-22T23:28:39.669ZLoading write count information from file "/tmp/test_repair_verify.data"
106032023-09-22T23:28:39.672ZRun Repair workload
106042023-09-22T23:28:39.675Z01/30 Write block 563 len 40960 data: 5 8 7 8 10 11 11 7 6 4
106052023-09-22T23:28:39.678Z02/30 Write block 165 len 4096 data: 6
106062023-09-22T23:28:39.681Z03/30 Write block 26 len 20480 data: 6 7 5 4 4
106072023-09-22T23:28:39.684Z04/30 Read block 164 len 8192
106082023-09-22T23:28:39.687Z05/30 Read block 425 len 24576
106092023-09-22T23:28:39.690Z06/30 Write block 92 len 32768 data: 6 5 5 8 8 7 9 8
106102023-09-22T23:28:39.694Z07/30 Write block 275 len 24576 data: 9 13 8 11 11 11
106112023-09-22T23:28:39.698Z08/30 Read block 269 len 28672
106122023-09-22T23:28:39.701Z09/30 Read block 324 len 8192
106132023-09-22T23:28:39.704Z10/30 Read block 414 len 40960
106142023-09-22T23:28:39.707Z11/30 Write block 459 len 16384 data: 8 7 8 7
106152023-09-22T23:28:39.710Z12/30 Read block 592 len 8192
106162023-09-22T23:28:39.713Z13/30 Read block 243 len 40960
106172023-09-22T23:28:39.716Z14/30 Write block 100 len 40960 data: 7 7 7 8 10 12 11 10 9 9
106182023-09-22T23:28:39.724Z15/30 Write block 524 len 16384 data: 6 7 8 8
106192023-09-22T23:28:39.727Z16/30 Write block 348 len 16384 data: 8 6 4 6
106202023-09-22T23:28:39.730Z17/30 Read block 150 len 32768
106212023-09-22T23:28:39.733Z18/30 Write block 275 len 40960 data: 10 14 9 12 12 12 10 10 10 8
106222023-09-22T23:28:39.735Z19/30 Read block 1 len 32768
106232023-09-22T23:28:39.738Z20/30 Write block 74 len 24576 data: 4 6 7 9 9 12
106242023-09-22T23:28:39.742Z21/30 Write block 465 len 36864 data: 11 9 9 9 8 6 4 4 4
106252023-09-22T23:28:39.745Z22/30 Read block 218 len 32768
106262023-09-22T23:28:39.748Z23/30 Read block 357 len 36864
106272023-09-22T23:28:39.751Z24/30 Write block 292 len 32768 data: 9 8 6 7 6 4 5 6
106282023-09-22T23:28:39.754Z25/30 Write block 191 len 24576 data: 7 6 6 7 7 7
106292023-09-22T23:28:39.756Z26/30 Read block 47 len 28672
106302023-09-22T23:28:39.759Z27/30 Write block 493 len 24576 data: 11 10 11 12 12 8
106312023-09-22T23:28:39.763Z28/30 Write block 137 len 28672 data: 8 8 8 8 8 7 8
106322023-09-22T23:28:39.766Z29/30 Read block 444 len 32768
106332023-09-22T23:28:39.769Z30/30 Write block 15 len 24576 data: 7 6 5 6 5 4
106342023-09-22T23:28:39.772Z----------------------------------------------------------------
106352023-09-22T23:28:39.775Z Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:30
106362023-09-22T23:28:39.779ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
106372023-09-22T23:28:39.781Z 1 Acked 1000 Write 10 Done Done Done false
106382023-09-22T23:28:39.784Z 2 Acked 1001 Write 1 Done Done Done false
106392023-09-22T23:28:39.787Z 3 Acked 1002 Write 5 Done Done Done false
106402023-09-22T23:28:39.790Z 4 Acked 1003 Read 2 Done Done Sent false
106412023-09-22T23:28:39.793Z 5 Acked 1004 Read 6 Done Done Sent false
106422023-09-22T23:28:39.796Z 6 Acked 1005 Write 8 Done Done Sent false
106432023-09-22T23:28:39.799Z 7 Acked 1006 Write 6 Done Done Sent false
106442023-09-22T23:28:39.802Z 8 Acked 1007 Read 7 Done Done Sent false
106452023-09-22T23:28:39.805Z 9 Acked 1008 Read 2 Done Done Sent false
106462023-09-22T23:28:39.808Z 10 Acked 1009 Read 10 Done Done Sent false
106472023-09-22T23:28:39.811Z 11 Acked 1010 Write 4 Done Done Sent false
106482023-09-22T23:28:39.814Z 12 Acked 1011 Read 2 Done Done Sent false
106492023-09-22T23:28:39.817Z 13 Acked 1012 Read 10 Done Done Sent false
106502023-09-22T23:28:39.822Z 14 Acked 1013 Write 10 Done Done Sent false
106512023-09-22T23:28:39.825Z 15 Acked 1014 Write 4 Done Done Sent false
106522023-09-22T23:28:39.828Z 16 Acked 1015 Write 4 Done Done Sent false
106532023-09-22T23:28:39.831Z 17 Acked 1016 Read 8 Done Done Sent false
106542023-09-22T23:28:39.834Z 18 Acked 1017 Write 10 Done Done Sent false
106552023-09-22T23:28:39.837Z 19 Acked 1018 Read 8 Done Done Sent false
106562023-09-22T23:28:39.840Z 20 Acked 1019 Write 6 Done Done Sent false
106572023-09-22T23:28:39.843Z 21 Acked 1020 Write 9 Done Done Sent false
106582023-09-22T23:28:39.846Z 22 Acked 1021 Read 8 Done Done Sent false
106592023-09-22T23:28:39.849Z 23 Acked 1022 Read 9 Done Done Sent false
106602023-09-22T23:28:39.852Z 24 Acked 1023 Write 8 Done Done Sent false
106612023-09-22T23:28:39.855Z 25 Acked 1024 Write 6 Done Done Sent false
106622023-09-22T23:28:39.857Z 26 Acked 1025 Read 7 Done Done Sent false
106632023-09-22T23:28:39.861Z 27 Acked 1026 Write 6 Done Done Sent false
106642023-09-22T23:28:39.864Z 28 Acked 1027 Write 7 Done Done Sent false
106652023-09-22T23:28:39.866Z 29 Acked 1028 Read 8 Done Done Sent false
106662023-09-22T23:28:39.869Z 30 Acked 1029 Write 6 Sent Sent Sent false
106672023-09-22T23:28:39.872Z STATES DS:0 DS:1 DS:2 TOTAL
106682023-09-22T23:28:39.876Z New 0 0 0 0
106692023-09-22T23:28:39.879Z Sent 1 1 27 29
106702023-09-22T23:28:39.881Z Done 29 29 3 61
106712023-09-22T23:28:39.884Z Skipped 0 0 0 0
106722023-09-22T23:28:39.887Z Error 0 0 0 0
106732023-09-22T23:28:39.890ZLast Flush: 0 0 0
106742023-09-22T23:28:39.898ZDownstairs last five completed:
106752023-09-22T23:28:39.901ZUpstairs last five completed: 30 29 28 27 26 25
106762023-09-22T23:28:39.906ZWrote out file "/tmp/test_repair_verify.data"
106772023-09-22T23:28:39.909Z
106782023-09-22T23:28:39.912ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
106792023-09-22T23:28:39.915Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.636163202Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1489}
106802023-09-22T23:28:39.918Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.636428247Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1489}
106812023-09-22T23:28:39.921Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.636451274Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1489}
106822023-09-22T23:28:39.924Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.680922036Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1489}
106832023-09-22T23:28:39.927Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.680956314Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1489}
106842023-09-22T23:28:39.931Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.680975658Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1489}
106852023-09-22T23:28:39.934Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.713973039Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1489}
106862023-09-22T23:28:39.937Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.71400625Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1489}
106872023-09-22T23:28:39.941Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:39.714025814Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1489}
106882023-09-22T23:28:39.946ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
106892023-09-22T23:28:39.949Z 0 000-019 60 60 60 90 90 90 T T F <---
106902023-09-22T23:28:39.952ZError: Difference in extent metadata found!
106912023-09-22T23:28:39.955Z 1 020-039 74 74 74 111 111 111 T T T
106922023-09-22T23:28:39.958Z 2 040-059 62 62 62 92 92 92 F F F
106932023-09-22T23:28:39.961Z 3 060-079 74 74 74 111 111 111 T T F <---
106942023-09-22T23:28:39.964Z 4 080-099 74 74 74 111 111 111 T T F <---
106952023-09-22T23:28:39.967Z 5 100-119 79 79 79 122 122 122 T T F <---
106962023-09-22T23:28:39.970Z 6 120-139 77 77 77 116 116 116 T T F <---
106972023-09-22T23:28:39.973Z 7 140-159 76 76 76 113 113 113 T T F <---
106982023-09-22T23:28:39.976Z 8 160-179 78 78 78 118 118 118 T T T
106992023-09-22T23:28:39.980Z 9 180-199 78 78 78 121 121 121 T T F <---
107002023-09-22T23:28:39.984Z 10 200-219 79 79 79 122 122 122 F F F
107012023-09-22T23:28:40.010Z 11 220-239 78 78 78 121 121 121 F F F
107022023-09-22T23:28:40.014Z 12 240-259 73 73 73 109 109 109 F F F
107032023-09-22T23:28:40.017Z 13 260-279 74 74 74 111 111 111 T T F <---
107042023-09-22T23:28:40.028Z 14 280-299 75 75 75 112 112 112 T T F <---
107052023-09-22T23:28:40.040Z 15 300-319 69 69 69 100 100 100 F F F
107062023-09-22T23:28:40.047Z 16 320-339 77 77 77 116 116 116 F F F
107072023-09-22T23:28:40.056Z 17 340-359 78 78 78 121 121 121 T T F <---
107082023-09-22T23:28:40.062Z 18 360-379 78 78 78 121 121 121 F F F
107092023-09-22T23:28:40.065Z 19 380-399 78 78 78 117 117 117 F F F
107102023-09-22T23:28:40.071Z 20 400-419 76 76 76 115 115 115 F F F
107112023-09-22T23:28:40.076Z 21 420-439 78 78 78 121 121 121 F F F
107122023-09-22T23:28:40.081Z 22 440-459 75 75 75 112 112 112 T T F <---
107132023-09-22T23:28:40.084Z 23 460-479 79 79 79 122 122 122 T T F <---
107142023-09-22T23:28:40.087Z 24 480-499 78 78 78 119 119 119 T T F <---
107152023-09-22T23:28:40.090Z 25 500-519 78 78 78 119 119 119 F F F
107162023-09-22T23:28:40.093Z 26 520-539 78 78 78 119 119 119 T T F <---
107172023-09-22T23:28:40.096Z 27 540-559 76 76 76 115 115 115 F F F
107182023-09-22T23:28:40.099Z 28 560-579 77 77 77 116 116 116 T T T
107192023-09-22T23:28:40.102Z 29 580-599 77 77 77 116 116 116 F F F
107202023-09-22T23:28:40.105ZMax gen: 79, Max flush: 122
107212023-09-22T23:28:40.108ZA Difference in extent metadata is expected here
107222023-09-22T23:28:40.115ZOn loop 40
107232023-09-22T23:28:40.118Z
107242023-09-22T23:28:40.121ZVerifying data now
107252023-09-22T23:28:46.504ZNo change after verify
107262023-09-22T23:28:46.507ZLoop: 40 Downstairs dump after verify (and repair):
107272023-09-22T23:28:46.522Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.521143719Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1494}
107282023-09-22T23:28:46.524Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.521552356Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1494}
107292023-09-22T23:28:46.528Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.521577422Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1494}
107302023-09-22T23:28:46.540Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.539988108Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1494}
107312023-09-22T23:28:46.543Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.54004123Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1494}
107322023-09-22T23:28:46.545Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.540062395Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1494}
107332023-09-22T23:28:46.551Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.550837344Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1494}
107342023-09-22T23:28:46.554Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.550889907Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1494}
107352023-09-22T23:28:46.557Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.5509107Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1494}
107362023-09-22T23:28:46.562ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
107372023-09-22T23:28:46.565Z 0 000-019 80 80 80 123 123 123 F F F
107382023-09-22T23:28:46.568Z 1 020-039 80 80 80 123 123 123 F F F
107392023-09-22T23:28:46.571Z 2 040-059 62 62 62 92 92 92 F F F
107402023-09-22T23:28:46.574Z 3 060-079 80 80 80 123 123 123 F F F
107412023-09-22T23:28:46.577Z 4 080-099 80 80 80 123 123 123 F F F
107422023-09-22T23:28:46.580Z 5 100-119 80 80 80 123 123 123 F F F
107432023-09-22T23:28:46.583Z 6 120-139 80 80 80 123 123 123 F F F
107442023-09-22T23:28:46.586Z 7 140-159 80 80 80 123 123 123 F F F
107452023-09-22T23:28:46.589Z 8 160-179 80 80 80 123 123 123 F F F
107462023-09-22T23:28:46.592Z 9 180-199 80 80 80 123 123 123 F F F
107472023-09-22T23:28:46.595Z 10 200-219 79 79 79 122 122 122 F F F
107482023-09-22T23:28:46.598Z 11 220-239 78 78 78 121 121 121 F F F
107492023-09-22T23:28:46.601Z 12 240-259 73 73 73 109 109 109 F F F
107502023-09-22T23:28:46.604Z 13 260-279 80 80 80 123 123 123 F F F
107512023-09-22T23:28:46.607Z 14 280-299 80 80 80 123 123 123 F F F
107522023-09-22T23:28:46.610Z 15 300-319 69 69 69 100 100 100 F F F
107532023-09-22T23:28:46.613Z 16 320-339 77 77 77 116 116 116 F F F
107542023-09-22T23:28:46.616Z 17 340-359 80 80 80 123 123 123 F F F
107552023-09-22T23:28:46.619Z 18 360-379 78 78 78 121 121 121 F F F
107562023-09-22T23:28:46.622Z 19 380-399 78 78 78 117 117 117 F F F
107572023-09-22T23:28:46.626Z 20 400-419 76 76 76 115 115 115 F F F
107582023-09-22T23:28:46.629Z 21 420-439 78 78 78 121 121 121 F F F
107592023-09-22T23:28:46.632Z 22 440-459 80 80 80 123 123 123 F F F
107602023-09-22T23:28:46.635Z 23 460-479 80 80 80 123 123 123 F F F
107612023-09-22T23:28:46.638Z 24 480-499 80 80 80 123 123 123 F F F
107622023-09-22T23:28:46.642Z 25 500-519 78 78 78 119 119 119 F F F
107632023-09-22T23:28:46.644Z 26 520-539 80 80 80 123 123 123 F F F
107642023-09-22T23:28:46.647Z 27 540-559 76 76 76 115 115 115 F F F
107652023-09-22T23:28:46.653Z 28 560-579 80 80 80 123 123 123 F F F
107662023-09-22T23:28:46.657Z 29 580-599 77 77 77 116 116 116 F F F
107672023-09-22T23:28:46.660ZMax gen: 80, Max flush: 123
107682023-09-22T23:28:46.663Z
107692023-09-22T23:28:46.667ZBegin loop 41
107702023-09-22T23:28:46.670ZDownstairs to restart: 2
107712023-09-22T23:28:46.673Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.597406526Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
107722023-09-22T23:28:46.676Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.597772605Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
107732023-09-22T23:28:46.679Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.597832389Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
107742023-09-22T23:28:46.682Z{"msg":"Crucible stats registered with UUID: 568d0d38-90cc-4d86-a79f-0792b1e104de","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.597873061Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
107752023-09-22T23:28:46.685Z{"msg":"Crucible 568d0d38-90cc-4d86-a79f-0792b1e104de has session id: 3facf057-826f-46c1-9fa1-1cf8c57cf7a8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.597919982Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
107762023-09-22T23:28:46.688ZCrucible runtime is spawned
107772023-09-22T23:28:46.691ZThe guest has requested activation with gen:82
107782023-09-22T23:28:46.693Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.59817756Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496,"task":"up_listen"}
107792023-09-22T23:28:46.696Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.598255358Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
107802023-09-22T23:28:46.699Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.59830586Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
107812023-09-22T23:28:46.703Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.598345945Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496,"looper":"1"}
107822023-09-22T23:28:46.706Z{"msg":"[1] 568d0d38-90cc-4d86-a79f-0792b1e104de looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.598656318Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496,"looper":"1"}
107832023-09-22T23:28:46.709Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.59872487Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496,"looper":"2"}
107842023-09-22T23:28:46.712Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.59889886Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496,"looper":"0"}
107852023-09-22T23:28:46.716Z{"msg":"[0] 568d0d38-90cc-4d86-a79f-0792b1e104de looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.599103681Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496,"looper":"0"}
107862023-09-22T23:28:46.719Z{"msg":"568d0d38-90cc-4d86-a79f-0792b1e104de active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.599412795Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
107872023-09-22T23:28:46.721Z{"msg":"Set desired generation to :82","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.599488604Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
107882023-09-22T23:28:46.724Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.599563895Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
107892023-09-22T23:28:46.727Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.599902524Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
107902023-09-22T23:28:46.730Z{"msg":"[1] 568d0d38-90cc-4d86-a79f-0792b1e104de (a1e0a7a3-fd0c-45b9-9671-188695470122) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.600258468Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
107912023-09-22T23:28:46.733Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.600330875Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
107922023-09-22T23:28:46.736Z{"msg":"[1] client is_active_req TRUE, promote! session a1e0a7a3-fd0c-45b9-9671-188695470122","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.600404231Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
107932023-09-22T23:28:46.739Z{"msg":"[0] 568d0d38-90cc-4d86-a79f-0792b1e104de (a1e0a7a3-fd0c-45b9-9671-188695470122) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.600483724Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
107942023-09-22T23:28:46.742Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.600554317Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
107952023-09-22T23:28:46.745Z{"msg":"[0] client is_active_req TRUE, promote! session a1e0a7a3-fd0c-45b9-9671-188695470122","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.600626686Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
107962023-09-22T23:28:46.748Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.600932052Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
107972023-09-22T23:28:46.752Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.601012438Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
107982023-09-22T23:28:46.755Z{"msg":"568d0d38-90cc-4d86-a79f-0792b1e104de WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.601082338Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
107992023-09-22T23:28:46.758Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.601168814Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108002023-09-22T23:28:46.761Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.601245851Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108012023-09-22T23:28:46.764Z{"msg":"568d0d38-90cc-4d86-a79f-0792b1e104de WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.601308344Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108022023-09-22T23:28:46.767Z{"msg":"[0] 568d0d38-90cc-4d86-a79f-0792b1e104de (a1e0a7a3-fd0c-45b9-9671-188695470122) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.603553974Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108032023-09-22T23:28:46.771Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.603633675Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108042023-09-22T23:28:46.775Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:28:46.603776488Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108052023-09-22T23:28:46.780Z{"msg":"[1] 568d0d38-90cc-4d86-a79f-0792b1e104de (a1e0a7a3-fd0c-45b9-9671-188695470122) WaitQuorum WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.603830544Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108062023-09-22T23:28:46.784Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.603871598Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108072023-09-22T23:28:46.791Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:28:46.603951607Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108082023-09-22T23:28:46.797Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.604009902Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108092023-09-22T23:28:46.801Z{"msg":"568d0d38-90cc-4d86-a79f-0792b1e104de WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.604054291Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108102023-09-22T23:28:46.805Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.604093311Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108112023-09-22T23:28:46.809Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.604131698Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108122023-09-22T23:28:46.813Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.604170582Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108132023-09-22T23:28:46.817Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.604208473Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108142023-09-22T23:28:46.821Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.604246768Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108152023-09-22T23:28:46.826Z{"msg":"568d0d38-90cc-4d86-a79f-0792b1e104de WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.604285371Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108162023-09-22T23:28:46.830Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.604305427Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108172023-09-22T23:28:46.833Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:46.604322883Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108182023-09-22T23:28:47.600Z{"msg":"[2] 568d0d38-90cc-4d86-a79f-0792b1e104de looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.599880827Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496,"looper":"2"}
108192023-09-22T23:28:47.603Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.599977098Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108202023-09-22T23:28:47.606Z{"msg":"[2] 568d0d38-90cc-4d86-a79f-0792b1e104de (a1e0a7a3-fd0c-45b9-9671-188695470122) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.601301226Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108212023-09-22T23:28:47.610Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.601372525Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108222023-09-22T23:28:47.612Z{"msg":"[2] client is_active_req TRUE, promote! session a1e0a7a3-fd0c-45b9-9671-188695470122","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.601396574Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108232023-09-22T23:28:47.615Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.601997488Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108242023-09-22T23:28:47.618Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.602054581Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108252023-09-22T23:28:47.623Z{"msg":"568d0d38-90cc-4d86-a79f-0792b1e104de WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.602100651Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108262023-09-22T23:28:47.626Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.604257796Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108272023-09-22T23:28:47.631Z{"msg":"[2] 568d0d38-90cc-4d86-a79f-0792b1e104de (a1e0a7a3-fd0c-45b9-9671-188695470122) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.60456614Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108282023-09-22T23:28:47.634Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.604622185Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108292023-09-22T23:28:47.637Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:28:47.604664372Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108302023-09-22T23:28:47.640Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.604810764Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108312023-09-22T23:28:47.643Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.604915139Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108322023-09-22T23:28:47.646Z{"msg":"568d0d38-90cc-4d86-a79f-0792b1e104de WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.604998449Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108332023-09-22T23:28:47.649Z{"msg":"The guest has finished waiting for activation with:82
108342023-09-22T23:28:47.652Z[0]R flush_numbers[0..12]: [123, 123, 92, 123, 123, 123, 123, 123, 123, 123, 122, 121]","v":0,"nameWait for a query_work_queue command to finish before sending IO
108352023-09-22T23:28:47.655Z":"crucible","level":30,"time":"2023-09-22T23:28:47.605219079Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108362023-09-22T23:28:47.658Z{"msg":"[0]R generation[0..12]: [80, 80, 62, 80, 80, 80, 80, 80, 80, 80, 79, 78]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.605301497Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108372023-09-22T23:28:47.661Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.605376995Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108382023-09-22T23:28:47.664Z{"msg":"[1]R flush_numbers[0..12]: [123, 123, 92, 123, 123, 123, 123, 123, 123, 123, 122, 121]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.605439649Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108392023-09-22T23:28:47.667Z{"msg":"[1]R generation[0..12]: [80, 80, 62, 80, 80, 80, 80, 80, 80, 80, 79, 78]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.60549125Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108402023-09-22T23:28:47.670Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.605532344Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108412023-09-22T23:28:47.673Z{"msg":"[2]R flush_numbers[0..12]: [123, 123, 92, 123, 123, 123, 123, 123, 123, 123, 122, 121]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.605572054Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108422023-09-22T23:28:47.676Z{"msg":"[2]R generation[0..12]: [80, 80, 62, 80, 80, 80, 80, 80, 80, 80, 79, 78]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.605615848Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108432023-09-22T23:28:47.679Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.605705994Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108442023-09-22T23:28:47.682Z{"msg":"Max found gen is 81","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.605753102Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108452023-09-22T23:28:47.685Z{"msg":"Generation requested: 82 >= found:81","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.605792754Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108462023-09-22T23:28:47.688Z{"msg":"Next flush: 124","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.605831136Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108472023-09-22T23:28:47.691Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.605868756Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108482023-09-22T23:28:47.694Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.605910589Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108492023-09-22T23:28:47.697Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.605956499Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108502023-09-22T23:28:47.699Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.605994614Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108512023-09-22T23:28:47.702Z{"msg":"568d0d38-90cc-4d86-a79f-0792b1e104de is now active with session: a1e0a7a3-fd0c-45b9-9671-188695470122","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.606034323Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108522023-09-22T23:28:47.706Z{"msg":"568d0d38-90cc-4d86-a79f-0792b1e104de Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.606073525Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108532023-09-22T23:28:47.709Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.60614684Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108542023-09-22T23:28:47.712Z{Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
108552023-09-22T23:28:47.715Z"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.606388292Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108562023-09-22T23:28:47.718Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.60646575Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108572023-09-22T23:28:47.721Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.606539818Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108582023-09-22T23:28:47.724Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.606601271Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108592023-09-22T23:28:47.727Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.606645412Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108602023-09-22T23:28:47.730Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.606686643Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108612023-09-22T23:28:47.733Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.606736536Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108622023-09-22T23:28:47.736Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.606778714Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108632023-09-22T23:28:47.739Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.60681712Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108642023-09-22T23:28:47.742Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.606855246Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1496}
108652023-09-22T23:28:47.745ZLoading write count information from file "/tmp/test_repair_verify.data"
108662023-09-22T23:28:47.748ZRun Repair workload
108672023-09-22T23:28:47.752Z01/30 Write block 576 len 20480 data: 7 8 7 6 6
108682023-09-22T23:28:47.756Z02/30 Write block 331 len 28672 data: 5 5 5 7 9 8 6
108692023-09-22T23:28:47.758Z03/30 Read block 9 len 4096
108702023-09-22T23:28:47.761Z04/30 Flush
108712023-09-22T23:28:47.764Z05/30 Write block 39 len 16384 data: 8 8 7 6
108722023-09-22T23:28:47.767Z06/30 Flush
108732023-09-22T23:28:47.770Z07/30 Flush
108742023-09-22T23:28:47.773Z08/30 Write block 247 len 36864 data: 4 3 5 5 5 8 7 8 7
108752023-09-22T23:28:47.776Z09/30 Read block 48 len 36864
108762023-09-22T23:28:47.779Z10/30 Read block 476 len 4096
108772023-09-22T23:28:47.782Z11/30 Read block 147 len 28672
108782023-09-22T23:28:47.784Z12/30 Read block 97 len 40960
108792023-09-22T23:28:47.787Z13/30 Write block 538 len 24576 data: 11 11 6 5 6 7
108802023-09-22T23:28:47.791Z14/30 Write block 311 len 16384 data: 2 2 4 6
108812023-09-22T23:28:47.794Z15/30 Write block 523 len 40960 data: 7 7 8 9 9 8 8 6 7 8
108822023-09-22T23:28:47.797Z16/30 Write block 284 len 40960 data: 9 9 9 9 9 7 8 9 10 9
108832023-09-22T23:28:47.800Z17/30 Read block 237 len 28672
108842023-09-22T23:28:47.803Z18/30 Write block 588 len 24576 data: 4 5 6 7 10 11
108852023-09-22T23:28:47.805Z19/30 Write block 415 len 28672 data: 7 8 8 7 8 8 8
108862023-09-22T23:28:47.808Z20/30 Write block 59 len 40960 data: 7 10 10 11 14 13 15 14 14 15
108872023-09-22T23:28:47.811Z21/30 Read block 80 len 8192
108882023-09-22T23:28:47.814Z22/30 Read block 591 len 20480
108892023-09-22T23:28:47.817Z23/30 Read block 555 len 20480
108902023-09-22T23:28:47.820Z24/30 Read block 422 len 8192
108912023-09-22T23:28:47.822Z25/30 Write block 326 len 12288 data: 6 4 6
108922023-09-22T23:28:47.826Z26/30 Write block 277 len 32768 data: 10 13 13 13 11 11 11 10
108932023-09-22T23:28:47.828Z27/30 Read block 356 len 20480
108942023-09-22T23:28:47.831Z28/30 Read block 429 len 32768
108952023-09-22T23:28:47.834Z29/30 Read block 375 len 8192
108962023-09-22T23:28:47.838Z30/30 Write block 122 len 40960 data: 8 8 7 6 4 7 7 6 6 5
108972023-09-22T23:28:47.841Z----------------------------------------------------------------
108982023-09-22T23:28:47.846Z Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30
108992023-09-22T23:28:47.850ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
109002023-09-22T23:28:47.853Z 1 Acked 1000 Write 5 Done Done Done false
109012023-09-22T23:28:47.856Z 2 Acked 1001 Write 7 Done Done Sent false
109022023-09-22T23:28:47.860Z 3 Acked 1002 Read 1 Done Done Sent false
109032023-09-22T23:28:47.862Z 4 Acked 1003 Flush 0 Done Done Sent false
109042023-09-22T23:28:47.865Z 5 Acked 1004 Write 4 Done Done Sent false
109052023-09-22T23:28:47.869Z 6 Acked 1005 Flush 0 Done Done Sent false
109062023-09-22T23:28:47.872Z 7 Acked 1006 Flush 0 Done Done Sent false
109072023-09-22T23:28:47.875Z 8 Acked 1007 Write 9 Done Done Sent false
109082023-09-22T23:28:48.037Z 9 Acked 1008 Read 9 Done Done Sent false
109092023-09-22T23:28:48.041Z 10 Acked 1009 Read 1 Done Done Sent false
109102023-09-22T23:28:48.045Z 11 Acked 1010 Read 7 Done Done Sent false
109112023-09-22T23:28:48.049Z 12 Acked 1011 Read 10 Done Done Sent false
109122023-09-22T23:28:48.053Z 13 Acked 1012 Write 6 Done Done Sent false
109132023-09-22T23:28:48.058Z 14 Acked 1013 Write 4 Done Done Sent false
109142023-09-22T23:28:48.062Z 15 Acked 1014 Write 10 Done Done Sent false
109152023-09-22T23:28:48.066Z 16 Acked 1015 Write 10 Done Done Sent false
109162023-09-22T23:28:48.070Z 17 Acked 1016 Read 7 Done Done Sent false
109172023-09-22T23:28:48.074Z 18 Acked 1017 Write 6 Done Done Sent false
109182023-09-22T23:28:48.078Z 19 Acked 1018 Write 7 Done Done Sent false
109192023-09-22T23:28:48.083Z 20 Acked 1019 Write 10 Done Done Sent false
109202023-09-22T23:28:48.087Z 21 Acked 1020 Read 2 Done Done Sent false
109212023-09-22T23:28:48.091Z 22 Acked 1021 Read 5 Done Done Sent false
109222023-09-22T23:28:48.095Z 23 Acked 1022 Read 5 Done Done Sent false
109232023-09-22T23:28:48.099Z 24 Acked 1023 Read 2 Done Done Sent false
109242023-09-22T23:28:48.103Z 25 Acked 1024 Write 3 Done Done Sent false
109252023-09-22T23:28:48.112Z 26 Acked 1025 Write 8 Done Done Sent false
109262023-09-22T23:28:48.117Z 27 Acked 1026 Read 5 Done Done Sent false
109272023-09-22T23:28:48.121Z 28 Acked 1027 Read 8 Done Done Sent false
109282023-09-22T23:28:48.124Z 29 Acked 1028 Read 2 Done Done Sent false
109292023-09-22T23:28:48.129Z 30 Acked 1029 Write 10 Sent Sent Sent false
109302023-09-22T23:28:48.132Z STATES DS:0 DS:1 DS:2 TOTAL
109312023-09-22T23:28:48.136Z New 0 0 0 0
109322023-09-22T23:28:48.140Z Sent 1 1 29 31
109332023-09-22T23:28:48.144Z Done 29 29 1 59
109342023-09-22T23:28:48.148Z Skipped 0 0 0 0
109352023-09-22T23:28:48.152Z Error 0 0 0 0
109362023-09-22T23:28:48.157ZLast Flush: 1006 1006 0
109372023-09-22T23:28:48.161ZDownstairs last five completed:
109382023-09-22T23:28:48.186ZUpstairs last five completed: 30 29 28 27 26 25
109392023-09-22T23:28:48.190ZWrote out file "/tmp/test_repair_verify.data"
109402023-09-22T23:28:48.194Z
109412023-09-22T23:28:48.198ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
109422023-09-22T23:28:48.202Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.747111192Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1497}
109432023-09-22T23:28:48.207Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.74747965Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1497}
109442023-09-22T23:28:48.212Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.747503417Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1497}
109452023-09-22T23:28:48.217Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.785134819Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1497}
109462023-09-22T23:28:48.221Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.785179271Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1497}
109472023-09-22T23:28:48.226Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.785198871Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1497}
109482023-09-22T23:28:48.230Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.812712652Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1497}
109492023-09-22T23:28:48.234Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.812773034Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1497}
109502023-09-22T23:28:48.238Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:47.812794956Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1497}
109512023-09-22T23:28:48.242ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
109522023-09-22T23:28:48.246Z 0 000-019 80 80 80 123 123 123 F F F
109532023-09-22T23:28:48.251Z 1 020-039 82 82 80 125 125 123 F F F <---
109542023-09-22T23:28:48.255Z 2 040-059 82 82 62 125 125 92 T T F <---
109552023-09-22T23:28:48.259Z 3 060-079 80 80 80 123 123 123 T T F <---
109562023-09-22T23:28:48.263Z 4 080-099 80 80 80 123 123 123 F F F
109572023-09-22T23:28:48.267Z 5 100-119 80 80 80 123 123 123 F F F
109582023-09-22T23:28:48.271Z 6 120-139 80 80 80 123 123 123 T T F <---
109592023-09-22T23:28:48.275Z 7 140-159 80 80 80 123 123 123 F F F
109602023-09-22T23:28:48.281Z 8 160-179 80 80 80 123 123 123 F F F
109612023-09-22T23:28:48.285ZError: Difference in extent metadata found!
109622023-09-22T23:28:48.290Z 9 180-199 80 80 80 123 123 123 F F F
109632023-09-22T23:28:48.294Z 10 200-219 79 79 79 122 122 122 F F F
109642023-09-22T23:28:48.298Z 11 220-239 78 78 78 121 121 121 F F F
109652023-09-22T23:28:48.302Z 12 240-259 73 73 73 109 109 109 T T F <---
109662023-09-22T23:28:48.306Z 13 260-279 80 80 80 123 123 123 T T F <---
109672023-09-22T23:28:48.311Z 14 280-299 80 80 80 123 123 123 T T F <---
109682023-09-22T23:28:48.315Z 15 300-319 69 69 69 100 100 100 T T F <---
109692023-09-22T23:28:48.319Z 16 320-339 82 82 77 124 124 116 T T F <---
109702023-09-22T23:28:48.323Z 17 340-359 80 80 80 123 123 123 F F F
109712023-09-22T23:28:48.327Z 18 360-379 78 78 78 121 121 121 F F F
109722023-09-22T23:28:48.331Z 19 380-399 78 78 78 117 117 117 F F F
109732023-09-22T23:28:48.335Z 20 400-419 76 76 76 115 115 115 T T F <---
109742023-09-22T23:28:48.340Z 21 420-439 78 78 78 121 121 121 T T F <---
109752023-09-22T23:28:48.344Z 22 440-459 80 80 80 123 123 123 F F F
109762023-09-22T23:28:48.347Z 23 460-479 80 80 80 123 123 123 F F F
109772023-09-22T23:28:48.352Z 24 480-499 80 80 80 123 123 123 F F F
109782023-09-22T23:28:48.356Z 25 500-519 78 78 78 119 119 119 F F F
109792023-09-22T23:28:48.362Z 26 520-539 80 80 80 123 123 123 T T F <---
109802023-09-22T23:28:48.365Z 27 540-559 76 76 76 115 115 115 T T F <---
109812023-09-22T23:28:48.369Z 28 560-579 82 82 80 124 124 123 F F T <---
109822023-09-22T23:28:48.372Z 29 580-599 82 82 77 124 124 116 T T T <---
109832023-09-22T23:28:48.377ZMax gen: 82, Max flush: 125
109842023-09-22T23:28:48.382ZA Difference in extent metadata is expected here
109852023-09-22T23:28:48.386ZOn loop 41
109862023-09-22T23:28:48.390Z
109872023-09-22T23:28:48.394ZVerifying data now
109882023-09-22T23:28:54.432ZNo change after verify
109892023-09-22T23:28:54.435ZLoop: 41 Downstairs dump after verify (and repair):
109902023-09-22T23:28:54.452Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.451065892Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1502}
109912023-09-22T23:28:54.455Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.451467142Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1502}
109922023-09-22T23:28:54.458Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.451507002Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1502}
109932023-09-22T23:28:54.472Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.47040516Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1502}
109942023-09-22T23:28:54.475Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.470467192Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1502}
109952023-09-22T23:28:54.479Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.470489445Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1502}
109962023-09-22T23:28:54.482Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.481520297Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1502}
109972023-09-22T23:28:54.485Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.481567634Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1502}
109982023-09-22T23:28:54.489Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.481609057Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1502}
109992023-09-22T23:28:54.493ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
110002023-09-22T23:28:54.496Z 0 000-019 80 80 80 123 123 123 F F F
110012023-09-22T23:28:54.500Z 1 020-039 82 82 82 125 125 125 F F F
110022023-09-22T23:28:54.505Z 2 040-059 83 83 83 126 126 126 F F F
110032023-09-22T23:28:54.509Z 3 060-079 83 83 83 126 126 126 F F F
110042023-09-22T23:28:54.512Z 4 080-099 80 80 80 123 123 123 F F F
110052023-09-22T23:28:54.518Z 5 100-119 80 80 80 123 123 123 F F F
110062023-09-22T23:28:54.521Z 6 120-139 83 83 83 126 126 126 F F F
110072023-09-22T23:28:54.525Z 7 140-159 80 80 80 123 123 123 F F F
110082023-09-22T23:28:54.535Z 8 160-179 80 80 80 123 123 123 F F F
110092023-09-22T23:28:54.539Z 9 180-199 80 80 80 123 123 123 F F F
110102023-09-22T23:28:54.543Z 10 200-219 79 79 79 122 122 122 F F F
110112023-09-22T23:28:54.550Z 11 220-239 78 78 78 121 121 121 F F F
110122023-09-22T23:28:54.555Z 12 240-259 83 83 83 126 126 126 F F F
110132023-09-22T23:28:54.559Z 13 260-279 83 83 83 126 126 126 F F F
110142023-09-22T23:28:54.562Z 14 280-299 83 83 83 126 126 126 F F F
110152023-09-22T23:28:54.568Z 15 300-319 83 83 83 126 126 126 F F F
110162023-09-22T23:28:54.586Z 16 320-339 83 83 83 126 126 126 F F F
110172023-09-22T23:28:55.006Z 17 340-359 80 80 80 123 123 123 F F F
110182023-09-22T23:28:55.009Z 18 360-379 78 78 78 121 121 121 F F F
110192023-09-22T23:28:55.012Z 19 380-399 78 78 78 117 117 117 F F F
110202023-09-22T23:28:55.042Z 20 400-419 83 83 83 126 126 126 F F F
110212023-09-22T23:28:55.046Z 21 420-439 83 83 83 126 126 126 F F F
110222023-09-22T23:28:55.049Z 22 440-459 80 80 80 123 123 123 F F F
110232023-09-22T23:28:55.052Z 23 460-479 80 80 80 123 123 123 F F F
110242023-09-22T23:28:55.054Z 24 480-499 80 80 80 123 123 123 F F F
110252023-09-22T23:28:55.057Z 25 500-519 78 78 78 119 119 119 F F F
110262023-09-22T23:28:55.060Z 26 520-539 83 83 83 126 126 126 F F F
110272023-09-22T23:28:55.064Z 27 540-559 83 83 83 126 126 126 F F F
110282023-09-22T23:28:55.067Z 28 560-579 82 82 82 124 124 124 F F F
110292023-09-22T23:28:55.070Z 29 580-599 83 83 83 126 126 126 F F F
110302023-09-22T23:28:55.073ZMax gen: 83, Max flush: 126
110312023-09-22T23:28:55.076Z
110322023-09-22T23:28:55.079ZBegin loop 42
110332023-09-22T23:28:55.082ZDownstairs to restart: 1
110342023-09-22T23:28:55.085Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.533278208Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110352023-09-22T23:28:55.088Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.533653375Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110362023-09-22T23:28:55.091Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.53370531Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110372023-09-22T23:28:55.098Z{"msg":"Crucible stats registered with UUID: f7d27ca0-181d-489e-b7ee-2ae2cbf4f054","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.533771043Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110382023-09-22T23:28:55.109Z{"msg":"Crucible f7d27ca0-181d-489e-b7ee-2ae2cbf4f054 has session id: 4427609e-0022-4a01-a7eb-1ae8ba68b423","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.533838876Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110392023-09-22T23:28:55.112ZCrucible runtime is spawned
110402023-09-22T23:28:55.116ZThe guest has requested activation with gen:84
110412023-09-22T23:28:55.119Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.534090767Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504,"task":"up_listen"}
110422023-09-22T23:28:55.122Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.534152048Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110432023-09-22T23:28:55.125Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.534206133Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110442023-09-22T23:28:55.128Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.534242356Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504,"looper":"2"}
110452023-09-22T23:28:55.131Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.534584153Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504,"looper":"1"}
110462023-09-22T23:28:55.137Z{"msg":"[2] f7d27ca0-181d-489e-b7ee-2ae2cbf4f054 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.534687928Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504,"looper":"2"}
110472023-09-22T23:28:55.140Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.534795889Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504,"looper":"0"}
110482023-09-22T23:28:55.143Z{"msg":"[0] f7d27ca0-181d-489e-b7ee-2ae2cbf4f054 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.535030291Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504,"looper":"0"}
110492023-09-22T23:28:55.146Z{"msg":"f7d27ca0-181d-489e-b7ee-2ae2cbf4f054 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.535128326Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110502023-09-22T23:28:55.149Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.535171923Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110512023-09-22T23:28:55.152Z{"msg":"Set desired generation to :84","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.535208959Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110522023-09-22T23:28:55.155Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.535308039Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110532023-09-22T23:28:55.159Z{"msg":"[0] f7d27ca0-181d-489e-b7ee-2ae2cbf4f054 (b1a0f1ee-7822-4ebe-9167-3f6a86b5b8d0) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.535976051Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110542023-09-22T23:28:55.162Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.536071575Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110552023-09-22T23:28:55.165Z{"msg":"[2] f7d27ca0-181d-489e-b7ee-2ae2cbf4f054 (b1a0f1ee-7822-4ebe-9167-3f6a86b5b8d0) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.536131542Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110562023-09-22T23:28:55.168Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.536228151Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110572023-09-22T23:28:55.171Z{"msg":"[0] client is_active_req TRUE, promote! session b1a0f1ee-7822-4ebe-9167-3f6a86b5b8d0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.536250518Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110582023-09-22T23:28:55.175Z{"msg":"[2] client is_active_req TRUE, promote! session b1a0f1ee-7822-4ebe-9167-3f6a86b5b8d0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.536294607Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110592023-09-22T23:28:55.180Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.53672675Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110602023-09-22T23:28:55.183Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.536788272Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110612023-09-22T23:28:55.186Z{"msg":"f7d27ca0-181d-489e-b7ee-2ae2cbf4f054 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.536881236Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110622023-09-22T23:28:55.189Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.536978948Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110632023-09-22T23:28:55.192Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.537057702Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110642023-09-22T23:28:55.195Z{"msg":"f7d27ca0-181d-489e-b7ee-2ae2cbf4f054 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.537131212Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110652023-09-22T23:28:55.198Z{"msg":"[0] f7d27ca0-181d-489e-b7ee-2ae2cbf4f054 (b1a0f1ee-7822-4ebe-9167-3f6a86b5b8d0) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.53921036Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110662023-09-22T23:28:55.202Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.539283302Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110672023-09-22T23:28:55.205Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:28:54.539327868Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110682023-09-22T23:28:55.208Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.539366765Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110692023-09-22T23:28:55.212Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.539406028Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110702023-09-22T23:28:55.215Z{"msg":"f7d27ca0-181d-489e-b7ee-2ae2cbf4f054 WaitQuorum New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.539445358Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110712023-09-22T23:28:55.218Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.539484465Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110722023-09-22T23:28:55.221Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.539522548Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110732023-09-22T23:28:55.224Z{"msg":"[2] f7d27ca0-181d-489e-b7ee-2ae2cbf4f054 (b1a0f1ee-7822-4ebe-9167-3f6a86b5b8d0) WaitQuorum New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.539563716Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110742023-09-22T23:28:55.227Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.539602788Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110752023-09-22T23:28:55.231Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:28:54.539641562Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110762023-09-22T23:28:55.234Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.539661326Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110772023-09-22T23:28:55.238Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.539679345Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110782023-09-22T23:28:55.248Z{"msg":"f7d27ca0-181d-489e-b7ee-2ae2cbf4f054 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.539697398Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110792023-09-22T23:28:55.251Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.539715191Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110802023-09-22T23:28:55.254Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:54.539732022Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110812023-09-22T23:28:55.538Z{"msg":"[1] f7d27ca0-181d-489e-b7ee-2ae2cbf4f054 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.536653512Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504,"looper":"1"}
110822023-09-22T23:28:55.541Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.536735914Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110832023-09-22T23:28:55.546Z{"msg":"[1] f7d27ca0-181d-489e-b7ee-2ae2cbf4f054 (b1a0f1ee-7822-4ebe-9167-3f6a86b5b8d0) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.537859692Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110842023-09-22T23:28:55.549Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.537914738Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110852023-09-22T23:28:55.553Z{"msg":"[1] client is_active_req TRUE, promote! session b1a0f1ee-7822-4ebe-9167-3f6a86b5b8d0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.537957954Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110862023-09-22T23:28:55.556Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.538563147Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110872023-09-22T23:28:55.578Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.538629316Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110882023-09-22T23:28:55.593Z{"msg":"f7d27ca0-181d-489e-b7ee-2ae2cbf4f054 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.538674092Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110892023-09-22T23:28:55.609Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.5407995Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110902023-09-22T23:28:55.624Z{"msg":"[1] f7d27ca0-181d-489e-b7ee-2ae2cbf4f054 (b1a0f1ee-7822-4ebe-9167-3f6a86b5b8d0) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.541050977Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110912023-09-22T23:28:55.644Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.541127184Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110922023-09-22T23:28:55.648Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:28:55.541194561Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110932023-09-22T23:28:55.651Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.541260571Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110942023-09-22T23:28:55.657Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.541320535Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110952023-09-22T23:28:55.663Z{"msg":"f7d27ca0-181d-489e-b7ee-2ae2cbf4f054 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.541363385Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110962023-09-22T23:28:55.690Z{"msg":"[0]R flush_numbers[0..12]: [123, 125, 126, 126, 123, 123, 126, 123, 123, 123, 122, 121]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.541405173Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
110972023-09-22T23:28:55.695Z{The guest has finished waiting for activation with:84
110982023-09-22T23:28:55.717Z"msg":"[0]R generation[0..12]: [80, 82, 83, 83, 80, 80, 83, 80, 80, 80, 79, 78]","v":0,"name":"Wait for a query_work_queue command to finish before sending IO
110992023-09-22T23:28:55.721Zcrucible","level":30,"time":"2023-09-22T23:28:55.54145301Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111002023-09-22T23:28:55.725Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.541494323Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111012023-09-22T23:28:55.729Z{"msg":"[1]R flush_numbers[0..12]: [123, 125, 126, 126, 123, 123, 126, 123, 123, 123, 122, 121]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.541534423Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111022023-09-22T23:28:55.733Z{"msg":"[1]R generation[0..12]: [80, 82, 83, 83, 80, 80, 83, 80, 80, 80, 79, 78]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.541589108Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111032023-09-22T23:28:55.751Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.541647975Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111042023-09-22T23:28:55.756Z{"msg":"[2]R flush_numbers[0..12]: [123, 125, 126, 126, 123, 123, 126, 123, 123, 123, 122, 121]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.54170389Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111052023-09-22T23:28:55.760Z{"msg":"[2]R generation[0..12]: [80, 82, 83, 83, 80, 80, 83, 80, 80, 80, 79, 78]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.541763042Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111062023-09-22T23:28:55.764Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.541824666Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111072023-09-22T23:28:55.769Z{"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.541880758Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111082023-09-22T23:28:55.773Z{"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.541936144Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111092023-09-22T23:28:55.789Z{"msg":"Next flush: 127","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.541989814Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111102023-09-22T23:28:55.795Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.542044052Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111112023-09-22T23:28:55.799Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.542108018Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111122023-09-22T23:28:55.804Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.54216221Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111132023-09-22T23:28:55.808Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.54223589Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111142023-09-22T23:28:55.812Z{"msg":"f7d27ca0-181d-489e-b7ee-2ae2cbf4f054 is now active with session: b1a0f1ee-7822-4ebe-9167-3f6a86b5b8d0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.542290679Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111152023-09-22T23:28:55.816Z{"msg":"f7d27ca0-181d-489e-b7ee-2ae2cbf4f054 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.542352204Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111162023-09-22T23:28:55.820Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.542414556Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111172023-09-22T23:28:55.823Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
111182023-09-22T23:28:55.827Z,"time":"2023-09-22T23:28:55.542480785Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111192023-09-22T23:28:55.830Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.542628303Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111202023-09-22T23:28:55.834Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.542684104Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111212023-09-22T23:28:55.837Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.542738011Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111222023-09-22T23:28:55.840Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.542790964Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111232023-09-22T23:28:55.845Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.54284288Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111242023-09-22T23:28:55.865Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.542895723Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111252023-09-22T23:28:55.877Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.542947738Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111262023-09-22T23:28:55.891Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.543000018Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111272023-09-22T23:28:55.895Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.543051924Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1504}
111282023-09-22T23:28:55.899ZLoading write count information from file "/tmp/test_repair_verify.data"
111292023-09-22T23:28:55.902ZRun Repair workload
111302023-09-22T23:28:55.905Z01/30 Write block 253 len 36864 data: 8 9 8 7 7 8 7 7 9
111312023-09-22T23:28:55.908Z02/30 Write block 142 len 4096 data: 8
111322023-09-22T23:28:55.911Z03/30 Write block 533 len 20480 data: 10 8 8 8 10
111332023-09-22T23:28:55.915Z04/30 Write block 568 len 12288 data: 12 12 8
111342023-09-22T23:28:55.939Z05/30 Read block 288 len 12288
111352023-09-22T23:28:55.970Z06/30 Flush
111362023-09-22T23:28:55.989Z07/30 Write block 200 len 40960 data: 6 5 8 8 7 11 12 11 13 11
111372023-09-22T23:28:56.013Z08/30 Read block 254 len 8192
111382023-09-22T23:28:56.020Z09/30 Write block 184 len 20480 data: 7 4 5 4 4
111392023-09-22T23:28:56.031Z10/30 Read block 371 len 40960
111402023-09-22T23:28:56.038Z11/30 Write block 559 len 8192 data: 6 6
111412023-09-22T23:28:56.044Z12/30 Read block 237 len 28672
111422023-09-22T23:28:56.063Z13/30 Write block 427 len 40960 data: 7 7 8 8 7 9 8 8 8 7
111432023-09-22T23:28:56.078Z14/30 Read block 33 len 24576
111442023-09-22T23:28:56.086Z15/30 Write block 267 len 32768 data: 10 8 9 9 10 10 9 9
111452023-09-22T23:28:56.094Z16/30 Read block 420 len 36864
111462023-09-22T23:28:56.098Z17/30 Write block 123 len 20480 data: 9 8 7 5 8
111472023-09-22T23:28:56.103Z18/30 Read block 262 len 20480
111482023-09-22T23:28:56.143Z19/30 Read block 44 len 24576
111492023-09-22T23:28:56.152Z20/30 Read block 219 len 28672
111502023-09-22T23:28:56.161Z21/30 Read block 315 len 16384
111512023-09-22T23:28:56.165Z22/30 Read block 445 len 36864
111522023-09-22T23:28:56.168Z23/30 Write block 450 len 20480 data: 9 9 9 8 7
111532023-09-22T23:28:56.172Z24/30 Write block 565 len 12288 data: 8 9 11
111542023-09-22T23:28:56.175Z25/30 Write block 286 len 8192 data: 10 10
111552023-09-22T23:28:56.179Z26/30 Write block 392 len 8192 data: 6 5
111562023-09-22T23:28:56.184Z27/30 Read block 395 len 36864
111572023-09-22T23:28:56.187Z28/30 Read block 44 len 4096
111582023-09-22T23:28:56.191Z29/30 Write block 92 len 24576 data: 7 6 6 9 9 8
111592023-09-22T23:28:56.194Z30/30 Read block 1 len 28672
111602023-09-22T23:28:56.198Z----------------------------------------------------------------
111612023-09-22T23:28:56.203Z Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:24
111622023-09-22T23:28:56.206ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
111632023-09-22T23:28:56.211Z 7 Acked 1006 Write 10 Done Sent Done false
111642023-09-22T23:28:56.215Z 8 Acked 1007 Read 2 Done Sent Done false
111652023-09-22T23:28:56.219Z 9 Acked 1008 Write 5 Done Sent Done false
111662023-09-22T23:28:56.236Z 10 Acked 1009 Read 10 Done Sent Done false
111672023-09-22T23:28:56.249Z 11 Acked 1010 Write 2 Done Sent Done false
111682023-09-22T23:28:56.274Z 12 Acked 1011 Read 7 Done Sent Done false
111692023-09-22T23:28:56.277Z 13 Acked 1012 Write 10 Done Sent Done false
111702023-09-22T23:28:56.281Z 14 Acked 1013 Read 6 Done Sent Done false
111712023-09-22T23:28:56.284Z 15 Acked 1014 Write 8 Done Sent Done false
111722023-09-22T23:28:56.287ZError: Difference in extent metadata found!
111732023-09-22T23:28:56.295Z 16 Acked 1015 Read 9 Done Sent Done false
111742023-09-22T23:28:56.298Z 17 Acked 1016 Write 5 Done Sent Done false
111752023-09-22T23:28:56.302Z 18 Acked 1017 Read 5 Done Sent Done false
111762023-09-22T23:28:56.305Z 19 Acked 1018 Read 6 Done Sent Done false
111772023-09-22T23:28:56.309Z 20 Acked 1019 Read 7 Done Sent Done false
111782023-09-22T23:28:56.313Z 21 Acked 1020 Read 4 Done Sent Done false
111792023-09-22T23:28:56.317Z 22 Acked 1021 Read 9 Done Sent Done false
111802023-09-22T23:28:56.321Z 23 Acked 1022 Write 5 Done Sent Done false
111812023-09-22T23:28:56.325Z 24 Acked 1023 Write 3 Done Sent Done false
111822023-09-22T23:28:56.328Z 25 Acked 1024 Write 2 Done Sent Done false
111832023-09-22T23:28:56.331Z 26 Acked 1025 Write 2 Done Sent Done false
111842023-09-22T23:28:56.335Z 27 Acked 1026 Read 9 Done Sent Done false
111852023-09-22T23:28:56.338Z 28 Acked 1027 Read 1 Done Sent Done false
111862023-09-22T23:28:56.342Z 29 Acked 1028 Write 6 Done Sent Done false
111872023-09-22T23:28:56.345Z 30 Acked 1029 Read 7 Done Sent Sent false
111882023-09-22T23:28:56.349Z STATES DS:0 DS:1 DS:2 TOTAL
111892023-09-22T23:28:56.353Z New 0 0 0 0
111902023-09-22T23:28:56.357Z Sent 0 24 1 25
111912023-09-22T23:28:56.361Z Done 24 0 23 47
111922023-09-22T23:28:56.364Z Skipped 0 0 0 0
111932023-09-22T23:28:56.368Z Error 0 0 0 0
111942023-09-22T23:28:56.372ZLast Flush: 1005 1005 1005
111952023-09-22T23:28:56.419ZDownstairs last five completed: 1005 1004 1003 1002 1001 1000
111962023-09-22T23:28:56.422ZUpstairs last five completed: 30 29 28 27 26 25
111972023-09-22T23:28:56.426ZWrote out file "/tmp/test_repair_verify.data"
111982023-09-22T23:28:56.432Z
111992023-09-22T23:28:56.436ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
112002023-09-22T23:28:56.439Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.68026989Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1505}
112012023-09-22T23:28:56.443Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.680590084Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1505}
112022023-09-22T23:28:56.447Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.680613321Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1505}
112032023-09-22T23:28:56.450Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.715563811Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1505}
112042023-09-22T23:28:56.454Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.715609979Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1505}
112052023-09-22T23:28:56.463Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.715628911Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1505}
112062023-09-22T23:28:56.467Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.74015674Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1505}
112072023-09-22T23:28:56.472Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.740201704Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1505}
112082023-09-22T23:28:56.479Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:28:55.740220895Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1505}
112092023-09-22T23:28:56.483ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
112102023-09-22T23:28:56.487Z 0 000-019 80 80 80 123 123 123 F F F
112112023-09-22T23:28:56.493Z 1 020-039 82 82 82 125 125 125 F F F
112122023-09-22T23:28:56.499Z 2 040-059 83 83 83 126 126 126 F F F
112132023-09-22T23:28:56.502Z 3 060-079 83 83 83 126 126 126 F F F
112142023-09-22T23:28:56.507Z 4 080-099 80 80 80 123 123 123 T F T <---
112152023-09-22T23:28:56.510Z 5 100-119 80 80 80 123 123 123 F F F
112162023-09-22T23:28:56.514Z 6 120-139 83 83 83 126 126 126 T F T <---
112172023-09-22T23:28:56.517Z 7 140-159 84 84 84 127 127 127 F F F
112182023-09-22T23:28:56.521Z 8 160-179 80 80 80 123 123 123 F F F
112192023-09-22T23:28:56.533Z 9 180-199 80 80 80 123 123 123 T F T <---
112202023-09-22T23:28:56.544Z 10 200-219 79 79 79 122 122 122 T F T <---
112212023-09-22T23:28:56.549Z 11 220-239 78 78 78 121 121 121 F F F
112222023-09-22T23:28:56.552Z 12 240-259 84 84 84 127 127 127 F F F
112232023-09-22T23:28:56.556Z 13 260-279 84 84 84 127 127 127 T F T <---
112242023-09-22T23:28:56.559Z 14 280-299 83 83 83 126 126 126 T F T <---
112252023-09-22T23:28:56.592Z 15 300-319 83 83 83 126 126 126 F F F
112262023-09-22T23:28:56.596Z 16 320-339 83 83 83 126 126 126 F F F
112272023-09-22T23:28:56.600Z 17 340-359 80 80 80 123 123 123 F F F
112282023-09-22T23:28:56.603Z 18 360-379 78 78 78 121 121 121 F F F
112292023-09-22T23:28:56.607Z 19 380-399 78 78 78 117 117 117 T F T <---
112302023-09-22T23:28:56.611Z 20 400-419 83 83 83 126 126 126 F F F
112312023-09-22T23:28:56.614Z 21 420-439 83 83 83 126 126 126 T F T <---
112322023-09-22T23:28:56.618Z 22 440-459 80 80 80 123 123 123 T F T <---
112332023-09-22T23:28:56.621Z 23 460-479 80 80 80 123 123 123 F F F
112342023-09-22T23:28:56.624Z 24 480-499 80 80 80 123 123 123 F F F
112352023-09-22T23:28:56.628Z 25 500-519 78 78 78 119 119 119 F F F
112362023-09-22T23:28:56.631Z 26 520-539 84 84 84 127 127 127 F F F
112372023-09-22T23:28:56.635Z 27 540-559 83 83 83 126 126 126 T F T <---
112382023-09-22T23:28:56.638Z 28 560-579 84 84 84 127 127 127 T F T <---
112392023-09-22T23:28:56.642Z 29 580-599 83 83 83 126 126 126 F F F
112402023-09-22T23:28:56.645ZMax gen: 84, Max flush: 127
112412023-09-22T23:28:56.649ZA Difference in extent metadata is expected here
112422023-09-22T23:28:56.653ZOn loop 42
112432023-09-22T23:28:56.657Z
112442023-09-22T23:28:56.660ZVerifying data now
112452023-09-22T23:29:02.032ZNo change after verify
112462023-09-22T23:29:02.035ZLoop: 42 Downstairs dump after verify (and repair):
112472023-09-22T23:29:02.051Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.050232524Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1510}
112482023-09-22T23:29:02.054Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.050598042Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1510}
112492023-09-22T23:29:02.057Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.050623118Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1510}
112502023-09-22T23:29:02.069Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.06929928Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1510}
112512023-09-22T23:29:02.073Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.069352444Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1510}
112522023-09-22T23:29:02.077Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.069373412Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1510}
112532023-09-22T23:29:02.081Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.080486837Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1510}
112542023-09-22T23:29:02.084Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.080538926Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1510}
112552023-09-22T23:29:02.088Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.080559583Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1510}
112562023-09-22T23:29:02.092ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
112572023-09-22T23:29:02.095Z 0 000-019 80 80 80 123 123 123 F F F
112582023-09-22T23:29:02.099Z 1 020-039 82 82 82 125 125 125 F F F
112592023-09-22T23:29:02.102Z 2 040-059 83 83 83 126 126 126 F F F
112602023-09-22T23:29:02.105Z 3 060-079 83 83 83 126 126 126 F F F
112612023-09-22T23:29:02.109Z 4 080-099 85 85 85 128 128 128 F F F
112622023-09-22T23:29:02.114Z 5 100-119 80 80 80 123 123 123 F F F
112632023-09-22T23:29:02.118Z 6 120-139 85 85 85 128 128 128 F F F
112642023-09-22T23:29:02.121Z 7 140-159 84 84 84 127 127 127 F F F
112652023-09-22T23:29:02.125Z 8 160-179 80 80 80 123 123 123 F F F
112662023-09-22T23:29:02.129Z 9 180-199 85 85 85 128 128 128 F F F
112672023-09-22T23:29:02.134Z 10 200-219 85 85 85 128 128 128 F F F
112682023-09-22T23:29:02.137Z 11 220-239 78 78 78 121 121 121 F F F
112692023-09-22T23:29:02.140Z 12 240-259 84 84 84 127 127 127 F F F
112702023-09-22T23:29:02.144Z 13 260-279 85 85 85 128 128 128 F F F
112712023-09-22T23:29:02.147Z 14 280-299 85 85 85 128 128 128 F F F
112722023-09-22T23:29:02.151Z 15 300-319 83 83 83 126 126 126 F F F
112732023-09-22T23:29:02.154Z 16 320-339 83 83 83 126 126 126 F F F
112742023-09-22T23:29:02.158Z 17 340-359 80 80 80 123 123 123 F F F
112752023-09-22T23:29:02.162Z 18 360-379 78 78 78 121 121 121 F F F
112762023-09-22T23:29:02.166Z 19 380-399 85 85 85 128 128 128 F F F
112772023-09-22T23:29:02.169Z 20 400-419 83 83 83 126 126 126 F F F
112782023-09-22T23:29:02.172Z 21 420-439 85 85 85 128 128 128 F F F
112792023-09-22T23:29:02.176Z 22 440-459 85 85 85 128 128 128 F F F
112802023-09-22T23:29:02.180Z 23 460-479 80 80 80 123 123 123 F F F
112812023-09-22T23:29:02.183Z 24 480-499 80 80 80 123 123 123 F F F
112822023-09-22T23:29:02.187Z 25 500-519 78 78 78 119 119 119 F F F
112832023-09-22T23:29:02.190Z 26 520-539 84 84 84 127 127 127 F F F
112842023-09-22T23:29:02.193Z 27 540-559 85 85 85 128 128 128 F F F
112852023-09-22T23:29:02.196Z 28 560-579 85 85 85 128 128 128 F F F
112862023-09-22T23:29:02.199Z 29 580-599 83 83 83 126 126 126 F F F
112872023-09-22T23:29:02.203ZMax gen: 85, Max flush: 128
112882023-09-22T23:29:02.206Z
112892023-09-22T23:29:02.210ZBegin loop 43
112902023-09-22T23:29:02.213ZDownstairs to restart: 1
112912023-09-22T23:29:02.216Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.128186309Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
112922023-09-22T23:29:02.220Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.128579443Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
112932023-09-22T23:29:02.223Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.128631999Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
112942023-09-22T23:29:02.227Z{"msg":"Crucible stats registered with UUID: 54475ccf-1669-4ea2-bf75-62e2c8398e74","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.128679717Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
112952023-09-22T23:29:02.231Z{"msg":"Crucible 54475ccf-1669-4ea2-bf75-62e2c8398e74 has session id: db9d5b4c-33dd-49e1-a398-048f196727e6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.128747937Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
112962023-09-22T23:29:02.235ZCrucible runtime is spawned
112972023-09-22T23:29:02.238Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30The guest has requested activation with gen:86
112982023-09-22T23:29:02.241Z,"time":"2023-09-22T23:29:02.129023921Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512,"task":"up_listen"}
112992023-09-22T23:29:02.245Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.129120168Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113002023-09-22T23:29:02.248Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.129201524Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113012023-09-22T23:29:02.251Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.129275912Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512,"looper":"2"}
113022023-09-22T23:29:02.255Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.129578744Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512,"looper":"1"}
113032023-09-22T23:29:02.258Z{"msg":"[2] 54475ccf-1669-4ea2-bf75-62e2c8398e74 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.12966706Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512,"looper":"2"}
113042023-09-22T23:29:02.262Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.12974708Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512,"looper":"0"}
113052023-09-22T23:29:02.266Z{"msg":"[0] 54475ccf-1669-4ea2-bf75-62e2c8398e74 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.12988922Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512,"looper":"0"}
113062023-09-22T23:29:02.269Z{"msg":"54475ccf-1669-4ea2-bf75-62e2c8398e74 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.130140008Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113072023-09-22T23:29:02.273Z{"msg":"Set desired generation to :86","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.130202807Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113082023-09-22T23:29:02.277Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.130313671Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113092023-09-22T23:29:02.283Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.130656367Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113102023-09-22T23:29:02.286Z{"msg":"[2] 54475ccf-1669-4ea2-bf75-62e2c8398e74 (5c2ac280-2415-483c-8af6-45139ccb74c0) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.131039031Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113112023-09-22T23:29:02.290Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.131125953Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113122023-09-22T23:29:02.293Z{"msg":"[2] client is_active_req TRUE, promote! session 5c2ac280-2415-483c-8af6-45139ccb74c0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.131150371Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113132023-09-22T23:29:02.297Z{"msg":"[0] 54475ccf-1669-4ea2-bf75-62e2c8398e74 (5c2ac280-2415-483c-8af6-45139ccb74c0) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.131191533Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113142023-09-22T23:29:02.300Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.13125261Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113152023-09-22T23:29:02.304Z{"msg":"[0] client is_active_req TRUE, promote! session 5c2ac280-2415-483c-8af6-45139ccb74c0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.131274093Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113162023-09-22T23:29:02.308Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.131754914Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113172023-09-22T23:29:02.314Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.131837548Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113182023-09-22T23:29:02.317Z{"msg":"54475ccf-1669-4ea2-bf75-62e2c8398e74 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.131909057Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113192023-09-22T23:29:02.322Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.131991198Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113202023-09-22T23:29:02.325Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.132073457Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113212023-09-22T23:29:02.330Z{"msg":"54475ccf-1669-4ea2-bf75-62e2c8398e74 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.132142527Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113222023-09-22T23:29:02.333Z{"msg":"[2] 54475ccf-1669-4ea2-bf75-62e2c8398e74 (5c2ac280-2415-483c-8af6-45139ccb74c0) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.133921529Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113232023-09-22T23:29:02.337Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.13400286Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113242023-09-22T23:29:02.345Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:29:02.134074645Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113252023-09-22T23:29:02.348Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.134148306Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113262023-09-22T23:29:02.352Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.134221467Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113272023-09-22T23:29:02.356Z{"msg":"54475ccf-1669-4ea2-bf75-62e2c8398e74 WaitActive New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.134274213Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113282023-09-22T23:29:02.364Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.13432327Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113292023-09-22T23:29:02.368Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.134362627Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113302023-09-22T23:29:02.375Z{"msg":"[0] 54475ccf-1669-4ea2-bf75-62e2c8398e74 (5c2ac280-2415-483c-8af6-45139ccb74c0) WaitActive New WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.135150706Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113312023-09-22T23:29:02.379Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.135231878Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113322023-09-22T23:29:02.382Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:29:02.135302867Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113332023-09-22T23:29:02.386Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.135349073Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113342023-09-22T23:29:02.391Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.13536964Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113352023-09-22T23:29:02.394Z{"msg":"54475ccf-1669-4ea2-bf75-62e2c8398e74 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.135388103Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113362023-09-22T23:29:02.398Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.135405632Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113372023-09-22T23:29:02.403Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:02.135423063Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113382023-09-22T23:29:03.132Z{"msg":"[1] 54475ccf-1669-4ea2-bf75-62e2c8398e74 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.131543336Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512,"looper":"1"}
113392023-09-22T23:29:03.137Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.131626985Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113402023-09-22T23:29:03.141Z{"msg":"[1] 54475ccf-1669-4ea2-bf75-62e2c8398e74 (5c2ac280-2415-483c-8af6-45139ccb74c0) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.132998367Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113412023-09-22T23:29:03.145Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.133094482Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113422023-09-22T23:29:03.148Z{"msg":"[1] client is_active_req TRUE, promote! session 5c2ac280-2415-483c-8af6-45139ccb74c0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.133152441Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113432023-09-22T23:29:03.151Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.133684134Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113442023-09-22T23:29:03.155Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.133781851Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113452023-09-22T23:29:03.159Z{"msg":"54475ccf-1669-4ea2-bf75-62e2c8398e74 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.133842824Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113462023-09-22T23:29:03.162Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.135949912Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113472023-09-22T23:29:03.165Z{"msg":"[1] 54475ccf-1669-4ea2-bf75-62e2c8398e74 (5c2ac280-2415-483c-8af6-45139ccb74c0) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.136177512Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113482023-09-22T23:29:03.168Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.136232248Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113492023-09-22T23:29:03.171Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:29:03.136310968Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113502023-09-22T23:29:03.175Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.136333213Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113512023-09-22T23:29:03.179Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.136371168Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113522023-09-22T23:29:03.183Z{"msg":"54475ccf-1669-4ea2-bf75-62e2c8398e74 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.136471124Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113532023-09-22T23:29:03.187Z{"msg":"[0]R flush_numbers[0..12]: [123, 125, 126, 126, 128, 123, 128, 127, 123, 128, 128, 121]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.136518307Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113542023-09-22T23:29:03.190Z{"msg":"[0]R generation[0..12]: [80, 82, 83, 83, 85, 80, 85, 84, 80, 85, 85, 78]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.136562075Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113552023-09-22T23:29:03.195Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.136603176Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113562023-09-22T23:29:03.199Z{"msg":"[1]R flush_numbers[0..12]: [123, 125, 126, 126, 128, 123, 128, 127, 123, 128, 128, 121]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:86
113572023-09-22T23:29:03.202ZWait for a query_work_queue command to finish before sending IO
113582023-09-22T23:29:03.205Z,"time":"2023-09-22T23:29:03.136665091Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113592023-09-22T23:29:03.208Z{"msg":"[1]R generation[0..12]: [80, 82, 83, 83, 85, 80, 85, 84, 80, 85, 85, 78]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.13689151Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113602023-09-22T23:29:03.212Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.136952869Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113612023-09-22T23:29:03.214Z{"msg":"[2]R flush_numbers[0..12]: [123, 125, 126, 126, 128, 123, 128, 127, 123, 128, 128, 121]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.136976394Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113622023-09-22T23:29:03.218Z{"msg":"[2]R generation[0..12]: [80, 82, 83, 83, 85, 80, 85, 84, 80, 85, 85, 78]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.137024175Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113632023-09-22T23:29:03.222Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.137047382Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113642023-09-22T23:29:03.226Z{"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.137090782Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113652023-09-22T23:29:03.230Z{"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.137131526Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113662023-09-22T23:29:03.233Z{"msg":"Next flush: 129","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.137151165Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113672023-09-22T23:29:03.236Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.137169846Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113682023-09-22T23:29:03.240Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.137210752Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113692023-09-22T23:29:03.244Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.13725207Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113702023-09-22T23:29:03.248Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.13730486Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113712023-09-22T23:29:03.253Z{"msg":"54475ccf-1669-4ea2-bf75-62e2c8398e74 is now active with session: 5c2ac280-2415-483c-8af6-45139ccb74c0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.137327988Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113722023-09-22T23:29:03.257Z{"msg":"54475ccf-1669-4ea2-bf75-62e2c8398e74 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.137365226Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113732023-09-22T23:29:03.262Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.137408671Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113742023-09-22T23:29:03.265Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.137446697Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113752023-09-22T23:29:03.269Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.137484792Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113762023-09-22T23:29:03.273Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.137504195Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113772023-09-22T23:29:03.276Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.137540488Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113782023-09-22T23:29:03.279Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.137559874Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113792023-09-22T23:29:03.283Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.137596135Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113802023-09-22T23:29:03.286Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.137633792Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113812023-09-22T23:29:03.290Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
113822023-09-22T23:29:03.293Z,"time":"2023-09-22T23:29:03.1376751Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113832023-09-22T23:29:03.297Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.137792937Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113842023-09-22T23:29:03.300Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.137838344Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1512}
113852023-09-22T23:29:03.304ZLoading write count information from file "/tmp/test_repair_verify.data"
113862023-09-22T23:29:03.308ZRun Repair workload
113872023-09-22T23:29:03.311Z01/30 Write block 157 len 8192 data: 4 7
113882023-09-22T23:29:03.314Z02/30 Write block 60 len 40960 data: 11 11 12 15 14 16 15 15 16 15
113892023-09-22T23:29:03.317Z03/30 Read block 155 len 16384
113902023-09-22T23:29:03.320Z04/30 Write block 495 len 12288 data: 12 13 13
113912023-09-22T23:29:03.324Z05/30 Read block 7 len 20480
113922023-09-22T23:29:03.327Z06/30 Flush
113932023-09-22T23:29:03.331Z07/30 Write block 110 len 36864 data: 10 7 6 6 4 4 5 6 5
113942023-09-22T23:29:03.334Z08/30 Read block 509 len 4096
113952023-09-22T23:29:03.338Z09/30 Read block 521 len 8192
113962023-09-22T23:29:03.342Z10/30 Write block 275 len 28672 data: 11 15 11 14 14 14 12
113972023-09-22T23:29:03.345Z11/30 Read block 478 len 8192
113982023-09-22T23:29:03.349Z12/30 Read block 198 len 32768
113992023-09-22T23:29:03.353Z13/30 Write block 91 len 16384 data: 7 8 7 7
114002023-09-22T23:29:03.356Z14/30 Write block 220 len 20480 data: 6 5 5 6 5
114012023-09-22T23:29:03.359Z15/30 Read block 291 len 32768
114022023-09-22T23:29:03.362Z16/30 Write block 154 len 40960 data: 6 5 4 5 8 9 9 9 9 8
114032023-09-22T23:29:03.365Z17/30 Read block 342 len 24576
114042023-09-22T23:29:03.368Z18/30 Write block 203 len 24576 data: 9 8 12 13 12 14
114052023-09-22T23:29:03.371Z19/30 Read block 353 len 20480
114062023-09-22T23:29:03.375Z20/30 Read block 187 len 16384
114072023-09-22T23:29:03.379Z21/30 Read block 298 len 36864
114082023-09-22T23:29:03.383Z22/30 Read block 84 len 4096
114092023-09-22T23:29:03.387Z23/30 Read block 262 len 28672
114102023-09-22T23:29:03.390Z24/30 Write block 579 len 28672 data: 7 7 6 6 6 6 5
114112023-09-22T23:29:03.394Z25/30 Write block 340 len 12288 data: 4 3 6
114122023-09-22T23:29:03.397Z26/30 Flush
114132023-09-22T23:29:03.401Z27/30 Write block 488 len 20480 data: 11 12 11 12 11
114142023-09-22T23:29:03.404Z28/30 Write block 248 len 4096 data: 4
114152023-09-22T23:29:03.407Z29/30 Write block 328 len 36864 data: 7 5 4 6 6 6 8 10 9
114162023-09-22T23:29:03.411Z30/30 Write block 343 len 4096 data: 6
114172023-09-22T23:29:03.414Z----------------------------------------------------------------
114182023-09-22T23:29:03.417Z Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:24
114192023-09-22T23:29:03.420ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
114202023-09-22T23:29:03.424Z 7 Acked 1006 Write 9 Done Done Done false
114212023-09-22T23:29:03.427Z 8 Acked 1007 Read 1 Done Done Done false
114222023-09-22T23:29:03.431Z 9 Acked 1008 Read 2 Done Done Done false
114232023-09-22T23:29:03.435Z 10 Acked 1009 Write 7 Done Sent Done false
114242023-09-22T23:29:03.439Z 11 Acked 1010 Read 2 Done Sent Done false
114252023-09-22T23:29:03.444Z 12 Acked 1011 Read 8 Done Sent Done false
114262023-09-22T23:29:03.447Z 13 Acked 1012 Write 4 Done Sent Done false
114272023-09-22T23:29:03.450Z 14 Acked 1013 Write 5 Done Sent Done false
114282023-09-22T23:29:03.453Z 15 Acked 1014 Read 8 Done Sent Done false
114292023-09-22T23:29:03.456Z 16 Acked 1015 Write 10 Done Sent Done false
114302023-09-22T23:29:03.460Z 17 Acked 1016 Read 6 Done Sent Done false
114312023-09-22T23:29:03.464Z 18 Acked 1017 Write 6 Done Sent Done false
114322023-09-22T23:29:03.467Z 19 Acked 1018 Read 5 Done Sent Done false
114332023-09-22T23:29:03.471Z 20 Acked 1019 Read 4 Done Sent Done false
114342023-09-22T23:29:03.474Z 21 Acked 1020 Read 9 Done Sent Done false
114352023-09-22T23:29:03.477Z 22 Acked 1021 Read 1 Done Sent Done false
114362023-09-22T23:29:03.480Z 23 Acked 1022 Read 7 Done Sent Done false
114372023-09-22T23:29:03.484Z 24 Acked 1023 Write 7 Done Sent Done false
114382023-09-22T23:29:03.488Z 25 Acked 1024 Write 3 Done Sent Done false
114392023-09-22T23:29:03.501Z 26 Acked 1025 Flush 0 Done Sent Done false
114402023-09-22T23:29:03.505Z 27 Acked 1026 Write 5 Done Sent Done false
114412023-09-22T23:29:03.508Z 28 Acked 1027 Write 1 Sent Sent Sent false
114422023-09-22T23:29:03.512Z 29 Acked 1028 Write 9 Sent Sent Sent false
114432023-09-22T23:29:03.515Z 30 Acked 1029 Write 1 Sent Sent Sent false
114442023-09-22T23:29:03.519Z STATES DS:0 DS:1 DS:2 TOTAL
114452023-09-22T23:29:03.523Z New 0 0 0 0
114462023-09-22T23:29:03.526Z Sent 3 21 3 27
114472023-09-22T23:29:03.530Z Done 21 3 21 45
114482023-09-22T23:29:03.534Z Skipped 0 0 0 0
114492023-09-22T23:29:03.537Z Error 0 0 0 0
114502023-09-22T23:29:03.540ZLast Flush: 1025 1005 1025
114512023-09-22T23:29:03.544ZDownstairs last five completed: 1005 1004 1003 1002 1001 1000
114522023-09-22T23:29:03.547ZUpstairs last five completed: 30 29 28 27 26 25
114532023-09-22T23:29:03.550ZWrote out file "/tmp/test_repair_verify.data"
114542023-09-22T23:29:03.553Z
114552023-09-22T23:29:03.556ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
114562023-09-22T23:29:03.560Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.269171332Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1513}
114572023-09-22T23:29:03.564Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.269531449Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1513}
114582023-09-22T23:29:03.567Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.269554702Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1513}
114592023-09-22T23:29:03.570Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.293353023Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1513}
114602023-09-22T23:29:03.575ZError: Difference in extent metadata found!
114612023-09-22T23:29:03.579Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.293385551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1513}
114622023-09-22T23:29:03.582Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.293404869Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1513}
114632023-09-22T23:29:03.586Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.318698163Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1513}
114642023-09-22T23:29:03.590Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.318733879Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1513}
114652023-09-22T23:29:03.594Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:03.318753619Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1513}
114662023-09-22T23:29:03.597ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
114672023-09-22T23:29:03.600Z 0 000-019 80 80 80 123 123 123 F F F
114682023-09-22T23:29:03.603Z 1 020-039 82 82 82 125 125 125 F F F
114692023-09-22T23:29:03.606Z 2 040-059 83 83 83 126 126 126 F F F
114702023-09-22T23:29:03.609Z 3 060-079 86 86 86 129 129 129 F F F
114712023-09-22T23:29:03.612Z 4 080-099 86 85 86 130 128 130 F F F <---
114722023-09-22T23:29:03.615Z 5 100-119 86 80 86 130 123 130 F T F <---
114732023-09-22T23:29:03.618Z 6 120-139 85 85 85 128 128 128 F F F
114742023-09-22T23:29:03.621Z 7 140-159 86 86 86 130 129 130 F F F <---
114752023-09-22T23:29:03.624Z 8 160-179 86 80 86 130 123 130 F F F <---
114762023-09-22T23:29:03.628Z 9 180-199 85 85 85 128 128 128 F F F
114772023-09-22T23:29:03.631Z 10 200-219 86 85 86 130 128 130 F F F <---
114782023-09-22T23:29:03.637Z 11 220-239 86 78 86 130 121 130 F F F <---
114792023-09-22T23:29:03.641Z 12 240-259 84 84 84 127 127 127 T F T <---
114802023-09-22T23:29:03.644Z 13 260-279 86 85 86 130 128 130 F F F <---
114812023-09-22T23:29:03.648Z 14 280-299 86 85 86 130 128 130 F F F <---
114822023-09-22T23:29:03.652Z 15 300-319 83 83 83 126 126 126 F F F
114832023-09-22T23:29:03.655Z 16 320-339 83 83 83 126 126 126 T F T <---
114842023-09-22T23:29:03.658Z 17 340-359 86 80 86 130 123 130 F F F <---
114852023-09-22T23:29:03.662Z 18 360-379 78 78 78 121 121 121 F F F
114862023-09-22T23:29:03.665Z 19 380-399 85 85 85 128 128 128 F F F
114872023-09-22T23:29:03.668Z 20 400-419 83 83 83 126 126 126 F F F
114882023-09-22T23:29:03.696Z 21 420-439 85 85 85 128 128 128 F F F
114892023-09-22T23:29:03.701Z 22 440-459 85 85 85 128 128 128 F F F
114902023-09-22T23:29:03.704Z 23 460-479 80 80 80 123 123 123 F F F
114912023-09-22T23:29:03.708Z 24 480-499 86 86 86 129 129 129 T F T <---
114922023-09-22T23:29:03.712Z 25 500-519 78 78 78 119 119 119 F F F
114932023-09-22T23:29:03.715Z 26 520-539 84 84 84 127 127 127 F F F
114942023-09-22T23:29:03.718Z 27 540-559 85 85 85 128 128 128 F F F
114952023-09-22T23:29:03.721Z 28 560-579 86 85 86 130 128 130 F F F <---
114962023-09-22T23:29:03.727Z 29 580-599 86 83 86 130 126 130 F F F <---
114972023-09-22T23:29:03.730ZMax gen: 86, Max flush: 130
114982023-09-22T23:29:03.734ZA Difference in extent metadata is expected here
114992023-09-22T23:29:03.737ZOn loop 43
115002023-09-22T23:29:03.741Z
115012023-09-22T23:29:03.744ZVerifying data now
115022023-09-22T23:29:09.818ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
115032023-09-22T23:29:09.821Z346d345
115042023-09-22T23:29:09.825Z< 5,
115052023-09-22T23:29:09.828Z348a348
115062023-09-22T23:29:09.831Z> 6,
115072023-09-22T23:29:09.834Zdiff found after verify
115082023-09-22T23:29:09.862ZLoop: 43 Downstairs dump after verify (and repair):
115092023-09-22T23:29:09.865Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.84525879Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1521}
115102023-09-22T23:29:09.868Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.845564857Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1521}
115112023-09-22T23:29:09.871Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.845608328Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1521}
115122023-09-22T23:29:09.874Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.864291698Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1521}
115132023-09-22T23:29:09.877Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.864340738Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1521}
115142023-09-22T23:29:09.880Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.864378206Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1521}
115152023-09-22T23:29:09.883Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.87547751Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1521}
115162023-09-22T23:29:09.886Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.875529549Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1521}
115172023-09-22T23:29:09.889Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.875568641Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1521}
115182023-09-22T23:29:09.892ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
115192023-09-22T23:29:09.895Z 0 000-019 80 80 80 123 123 123 F F F
115202023-09-22T23:29:09.898Z 1 020-039 82 82 82 125 125 125 F F F
115212023-09-22T23:29:09.901Z 2 040-059 83 83 83 126 126 126 F F F
115222023-09-22T23:29:09.904Z 3 060-079 86 86 86 129 129 129 F F F
115232023-09-22T23:29:09.907Z 4 080-099 86 86 86 130 130 130 F F F
115242023-09-22T23:29:09.909Z 5 100-119 86 86 86 130 130 130 F F F
115252023-09-22T23:29:09.912Z 6 120-139 85 85 85 128 128 128 F F F
115262023-09-22T23:29:09.915Z 7 140-159 86 86 86 130 130 130 F F F
115272023-09-22T23:29:09.918Z 8 160-179 86 86 86 130 130 130 F F F
115282023-09-22T23:29:09.921Z 9 180-199 85 85 85 128 128 128 F F F
115292023-09-22T23:29:09.925Z 10 200-219 86 86 86 130 130 130 F F F
115302023-09-22T23:29:09.928Z 11 220-239 86 86 86 130 130 130 F F F
115312023-09-22T23:29:09.930Z 12 240-259 87 87 87 131 131 131 F F F
115322023-09-22T23:29:09.933Z 13 260-279 86 86 86 130 130 130 F F F
115332023-09-22T23:29:09.936Z 14 280-299 86 86 86 130 130 130 F F F
115342023-09-22T23:29:09.939Z 15 300-319 83 83 83 126 126 126 F F F
115352023-09-22T23:29:09.943Z 16 320-339 87 87 87 131 131 131 F F F
115362023-09-22T23:29:09.946Z 17 340-359 86 86 86 130 130 130 F F F
115372023-09-22T23:29:09.949Z 18 360-379 78 78 78 121 121 121 F F F
115382023-09-22T23:29:09.952Z 19 380-399 85 85 85 128 128 128 F F F
115392023-09-22T23:29:09.955Z 20 400-419 83 83 83 126 126 126 F F F
115402023-09-22T23:29:09.958Z 21 420-439 85 85 85 128 128 128 F F F
115412023-09-22T23:29:09.961Z 22 440-459 85 85 85 128 128 128 F F F
115422023-09-22T23:29:09.964Z 23 460-479 80 80 80 123 123 123 F F F
115432023-09-22T23:29:09.967Z 24 480-499 87 87 87 131 131 131 F F F
115442023-09-22T23:29:09.970Z 25 500-519 78 78 78 119 119 119 F F F
115452023-09-22T23:29:09.972Z 26 520-539 84 84 84 127 127 127 F F F
115462023-09-22T23:29:09.975Z 27 540-559 85 85 85 128 128 128 F F F
115472023-09-22T23:29:09.978Z 28 560-579 86 86 86 130 130 130 F F F
115482023-09-22T23:29:09.981Z 29 580-599 86 86 86 130 130 130 F F F
115492023-09-22T23:29:09.984ZMax gen: 87, Max flush: 131
115502023-09-22T23:29:09.987Z
115512023-09-22T23:29:09.990ZBegin loop 44
115522023-09-22T23:29:09.993ZDownstairs to restart: 1
115532023-09-22T23:29:09.996Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.923600377Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115542023-09-22T23:29:09.999Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.924002658Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115552023-09-22T23:29:10.002Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.924068892Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115562023-09-22T23:29:10.005Z{"msg":"Crucible stats registered with UUID: c73a3433-5be3-4f0e-a381-f4648ae853f9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.924094018Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115572023-09-22T23:29:10.008Z{"msg":"Crucible c73a3433-5be3-4f0e-a381-f4648ae853f9 has session id: 68ce1863-7cc3-481e-a349-b487f554d7ce","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.924136018Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115582023-09-22T23:29:10.012ZCrucible runtime is spawned
115592023-09-22T23:29:10.015ZThe guest has requested activation with gen:88
115602023-09-22T23:29:10.018Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.924465565Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523,"task":"up_listen"}
115612023-09-22T23:29:10.021Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.924560194Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115622023-09-22T23:29:10.024Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.924637431Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115632023-09-22T23:29:10.027Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.924716854Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523,"looper":"2"}
115642023-09-22T23:29:10.030Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.924813232Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523,"looper":"0"}
115652023-09-22T23:29:10.034Z{"msg":"[2] c73a3433-5be3-4f0e-a381-f4648ae853f9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.924920815Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523,"looper":"2"}
115662023-09-22T23:29:10.037Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.924992683Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523,"looper":"1"}
115672023-09-22T23:29:10.040Z{"msg":"[0] c73a3433-5be3-4f0e-a381-f4648ae853f9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.925032128Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523,"looper":"0"}
115682023-09-22T23:29:10.043Z{"msg":"c73a3433-5be3-4f0e-a381-f4648ae853f9 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.925455781Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115692023-09-22T23:29:10.046Z{"msg":"Set desired generation to :88","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.925516251Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115702023-09-22T23:29:10.049Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.925589529Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115712023-09-22T23:29:10.052Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.925908549Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115722023-09-22T23:29:10.055Z{"msg":"[2] c73a3433-5be3-4f0e-a381-f4648ae853f9 (e84ddc42-e147-4b50-afcd-b58ac850d84a) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.9263147Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115732023-09-22T23:29:10.058Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.926410429Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115742023-09-22T23:29:10.061Z{"msg":"[2] client is_active_req TRUE, promote! session e84ddc42-e147-4b50-afcd-b58ac850d84a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.926489581Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115752023-09-22T23:29:10.064Z{"msg":"[0] c73a3433-5be3-4f0e-a381-f4648ae853f9 (e84ddc42-e147-4b50-afcd-b58ac850d84a) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.926567807Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115762023-09-22T23:29:10.067Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.92664867Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115772023-09-22T23:29:10.069Z{"msg":"[0] client is_active_req TRUE, promote! session e84ddc42-e147-4b50-afcd-b58ac850d84a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.926735298Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115782023-09-22T23:29:10.072Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.926930413Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115792023-09-22T23:29:10.075Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.927047622Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115802023-09-22T23:29:10.078Z{"msg":"c73a3433-5be3-4f0e-a381-f4648ae853f9 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.927121516Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115812023-09-22T23:29:10.081Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.927194983Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115822023-09-22T23:29:10.084Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.927274158Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115832023-09-22T23:29:10.087Z{"msg":"c73a3433-5be3-4f0e-a381-f4648ae853f9 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.92734765Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115842023-09-22T23:29:10.090Z{"msg":"[0] c73a3433-5be3-4f0e-a381-f4648ae853f9 (e84ddc42-e147-4b50-afcd-b58ac850d84a) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.929630553Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115852023-09-22T23:29:10.094Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.929691534Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115862023-09-22T23:29:10.097Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:29:09.929749934Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115872023-09-22T23:29:10.099Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.929834699Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115882023-09-22T23:29:10.102Z{"msg":"[2] c73a3433-5be3-4f0e-a381-f4648ae853f9 (e84ddc42-e147-4b50-afcd-b58ac850d84a) WaitQuorum New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.929898011Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115892023-09-22T23:29:10.105Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.92994294Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115902023-09-22T23:29:10.108Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:29:09.92998242Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115912023-09-22T23:29:10.111Z{"msg":"c73a3433-5be3-4f0e-a381-f4648ae853f9 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.930022131Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115922023-09-22T23:29:10.114Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.930080649Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115932023-09-22T23:29:10.117Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.930125252Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115942023-09-22T23:29:10.120Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.930164012Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115952023-09-22T23:29:10.123Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.930201952Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115962023-09-22T23:29:10.126Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.930241762Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115972023-09-22T23:29:10.129Z{"msg":"c73a3433-5be3-4f0e-a381-f4648ae853f9 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.930262417Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115982023-09-22T23:29:10.132Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.930280131Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
115992023-09-22T23:29:10.135Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:09.93029706Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116002023-09-22T23:29:10.933Z{"msg":"[1] c73a3433-5be3-4f0e-a381-f4648ae853f9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.92719061Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523,"looper":"1"}
116012023-09-22T23:29:10.937Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.927284354Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116022023-09-22T23:29:10.941Z{"msg":"[1] c73a3433-5be3-4f0e-a381-f4648ae853f9 (e84ddc42-e147-4b50-afcd-b58ac850d84a) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.928323016Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116032023-09-22T23:29:10.944Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.92838019Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116042023-09-22T23:29:10.948Z{"msg":"[1] client is_active_req TRUE, promote! session e84ddc42-e147-4b50-afcd-b58ac850d84a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.928424133Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116052023-09-22T23:29:10.951Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.92901726Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116062023-09-22T23:29:10.955Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.929078Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116072023-09-22T23:29:10.958Z{"msg":"c73a3433-5be3-4f0e-a381-f4648ae853f9 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.929120442Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116082023-09-22T23:29:10.961Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.931477128Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116092023-09-22T23:29:10.964Z{"msg":"[1] c73a3433-5be3-4f0e-a381-f4648ae853f9 (e84ddc42-e147-4b50-afcd-b58ac850d84a) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.931633584Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116102023-09-22T23:29:10.968Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.931708714Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116112023-09-22T23:29:10.971Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:29:10.931754607Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116122023-09-22T23:29:10.974Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.93179436Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116132023-09-22T23:29:10.978Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.931834127Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116142023-09-22T23:29:10.981Z{"msg":"c73a3433-5be3-4f0e-a381-f4648ae853f9 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.931874498Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116152023-09-22T23:29:10.984Z{"msg":"[0]R flush_numbers[0..12]: [123, 125, 126, 129, 130, 130, 128, 130, 130, 128, 130, 130]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.931915856Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116162023-09-22T23:29:10.987Z{"msg":"[0]R generation[0..12]: [80, 82, 83, 86, 86, 86, 85, 86, 86, 85, 86, 86]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.931956018Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116172023-09-22T23:29:10.990Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.932010997Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116182023-09-22T23:29:10.993Z{"msg":"[1]R flush_numbers[0..12]: [123, 125, 126, 129, 130, 130, 128, 130, 130, 128, 130, 130]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.932035708Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116192023-09-22T23:29:10.997Z{"msg":"[1]R generation[0..12]: [80, 82, 83, 86, 86, 86, 85, 86, 86, 85, 86, 86]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.93205448Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116202023-09-22T23:29:11.001Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.932120146Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116212023-09-22T23:29:11.004Z{"msg":"[2]R flush_numbers[0..12]: [123, 125, 126, 129, 130, 130, 128, 130, 130, 128, 130, 130]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.932141326Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116222023-09-22T23:29:11.007Z{"msg":"[2]R generation[0..12]: [80, 82, 83, 86, 86, 86, 85, 86, 86, 85, 86, 86]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.932173064Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116232023-09-22T23:29:11.011ZThe guest has finished waiting for activation with:88
116242023-09-22T23:29:11.015Z{"msg":"Wait for a query_work_queue command to finish before sending IO
116252023-09-22T23:29:11.018Z[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.932213673Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116262023-09-22T23:29:11.021Z{"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.932247499Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116272023-09-22T23:29:11.025Z{"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.932281421Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116282023-09-22T23:29:11.029Z{"msg":"Next flush: 132","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.932334843Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116292023-09-22T23:29:11.032Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.932399195Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116302023-09-22T23:29:11.035Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.932464086Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116312023-09-22T23:29:11.039Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.932516495Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116322023-09-22T23:29:11.042Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.932538604Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116332023-09-22T23:29:11.045Z{"msg":"c73a3433-5be3-4f0e-a381-f4648ae853f9 is now active with session: e84ddc42-e147-4b50-afcd-b58ac850d84a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.932589553Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116342023-09-22T23:29:11.048Z{"msg":"c73a3433-5be3-4f0e-a381-f4648ae853f9 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.93265451Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116352023-09-22T23:29:11.051Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.932717194Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116362023-09-22T23:29:11.055Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.932761114Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116372023-09-22T23:29:11.058Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.932815032Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116382023-09-22T23:29:11.061Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.932882076Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116392023-09-22T23:29:11.064Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.932950179Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116402023-09-22T23:29:11.067Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.932993655Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116412023-09-22T23:29:11.070Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.933017854Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116422023-09-22T23:29:11.073Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.933080398Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116432023-09-22T23:29:11.076Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.933144074Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116442023-09-22T23:29:11.080Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.933187102Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116452023-09-22T23:29:11.083Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:10.9332136Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1523}
116462023-09-22T23:29:11.086ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
116472023-09-22T23:29:11.090ZLoading write count information from file "/tmp/test_repair_verify.data"
116482023-09-22T23:29:11.094ZRun Repair workload
116492023-09-22T23:29:11.098Z01/30 Write block 145 len 32768 data: 6 4 4 7 8 8 9 8
116502023-09-22T23:29:11.101Z02/30 Read block 112 len 24576
116512023-09-22T23:29:11.104Z03/30 Flush
116522023-09-22T23:29:11.107Z04/30 Flush
116532023-09-22T23:29:11.110Z05/30 Write block 430 len 24576 data: 9 8 10 9 9 9
116542023-09-22T23:29:11.121Z06/30 Read block 394 len 24576
116552023-09-22T23:29:11.125Z07/30 Read block 574 len 36864
116562023-09-22T23:29:11.138Z08/30 Read block 536 len 24576
116572023-09-22T23:29:11.150Z09/30 Read block 593 len 12288
116582023-09-22T23:29:11.153Z10/30 Read block 201 len 40960
116592023-09-22T23:29:11.156Z11/30 Read block 224 len 36864
116602023-09-22T23:29:11.159Z12/30 Write block 378 len 28672 data: 9 8 8 8 7 6 6
116612023-09-22T23:29:11.162Z13/30 Read block 561 len 36864
116622023-09-22T23:29:11.166Z14/30 Read block 485 len 20480
116632023-09-22T23:29:11.169Z15/30 Write block 37 len 32768 data: 10 10 9 9 8 7 8 8
116642023-09-22T23:29:11.172Z16/30 Write block 195 len 28672 data: 8 8 8 6 5 7 6
116652023-09-22T23:29:11.175Z17/30 Write block 459 len 24576 data: 9 8 9 8 8 8
116662023-09-22T23:29:11.179Z18/30 Write block 586 len 24576 data: 3 3 5 6 7 8
116672023-09-22T23:29:11.182Z19/30 Write block 447 len 16384 data: 6 8 9 10
116682023-09-22T23:29:11.184Z20/30 Flush
116692023-09-22T23:29:11.187Z21/30 Write block 359 len 20480 data: 7 8 7 6 6
116702023-09-22T23:29:11.190Z22/30 Write block 233 len 32768 data: 3 2 3 4 5 6 7 7
116712023-09-22T23:29:11.194Z23/30 Read block 358 len 40960
116722023-09-22T23:29:11.197Z24/30 Read block 79 len 40960
116732023-09-22T23:29:11.200Z25/30 Flush
116742023-09-22T23:29:11.203Z26/30 Flush
116752023-09-22T23:29:11.207Z27/30 Write block 258 len 40960 data: 9 8 8 10 10 10 10 8 8 11
116762023-09-22T23:29:11.210Z28/30 Write block 49 len 32768 data: 7 6 7 5 3 2 3 3
116772023-09-22T23:29:11.213Z29/30 Read block 324 len 24576
116782023-09-22T23:29:11.216Z30/30 Read block 205 len 32768
116792023-09-22T23:29:11.219Z----------------------------------------------------------------
116802023-09-22T23:29:11.222Z Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:26
116812023-09-22T23:29:11.225ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
116822023-09-22T23:29:11.228Z 5 Acked 1004 Write 6 Done Sent Done false
116832023-09-22T23:29:11.231Z 6 Acked 1005 Read 6 Done Sent Done false
116842023-09-22T23:29:11.234Z 7 Acked 1006 Read 9 Done Sent Done false
116852023-09-22T23:29:11.237Z 8 Acked 1007 Read 6 Done Sent Done false
116862023-09-22T23:29:11.241Z 9 Acked 1008 Read 3 Done Sent Done false
116872023-09-22T23:29:11.244Z 10 Acked 1009 Read 10 Done Sent Done false
116882023-09-22T23:29:11.248Z 11 Acked 1010 Read 9 Done Sent Done false
116892023-09-22T23:29:11.251Z 12 Acked 1011 Write 7 Done Sent Done false
116902023-09-22T23:29:11.255Z 13 Acked 1012 Read 9 Done Sent Done false
116912023-09-22T23:29:11.258Z 14 Acked 1013 Read 5 Done Sent Done false
116922023-09-22T23:29:11.262Z 15 Acked 1014 Write 8 Done Sent Done false
116932023-09-22T23:29:11.265Z 16 Acked 1015 Write 7 Done Sent Done false
116942023-09-22T23:29:11.268Z 17 Acked 1016 Write 6 Done Sent Done false
116952023-09-22T23:29:11.272Z 18 Acked 1017 Write 6 Done Sent Done false
116962023-09-22T23:29:11.275Z 19 Acked 1018 Write 4 Done Sent Done false
116972023-09-22T23:29:11.278Z 20 Acked 1019 Flush 0 Done Sent Done false
116982023-09-22T23:29:11.281Z 21 Acked 1020 Write 5 Done Sent Done false
116992023-09-22T23:29:11.285Z 22 Acked 1021 Write 8 Done Sent Done false
117002023-09-22T23:29:11.289Z 23 Acked 1022 Read 10 Done Sent Done false
117012023-09-22T23:29:11.292Z 24 Acked 1023 Read 10 Done Sent Done false
117022023-09-22T23:29:11.296Z 25 Acked 1024 Flush 0 Done Sent Done false
117032023-09-22T23:29:11.299Z 26 Acked 1025 Flush 0 Done Sent Done false
117042023-09-22T23:29:11.303Z 27 Acked 1026 Write 10 Done Sent Done false
117052023-09-22T23:29:11.306Z 28 Acked 1027 Write 8 Done Sent Done false
117062023-09-22T23:29:11.309Z 29 Acked 1028 Read 6 Done Sent Done false
117072023-09-22T23:29:11.312Z 30 Acked 1029 Read 8 Done Sent Done false
117082023-09-22T23:29:11.315Z STATES DS:0 DS:1 DS:2 TOTAL
117092023-09-22T23:29:11.318Z New 0 0 0 0
117102023-09-22T23:29:11.321Z Sent 0 26 0 26
117112023-09-22T23:29:11.324Z Done 26 0 26 52
117122023-09-22T23:29:11.327Z Skipped 0 0 0 0
117132023-09-22T23:29:11.331Z Error 0 0 0 0
117142023-09-22T23:29:11.334ZLast Flush: 1025 1003 1025
117152023-09-22T23:29:11.337ZDownstairs last five completed: 1003 1002 1001 1000
117162023-09-22T23:29:11.340ZUpstairs last five completed: 30 29 28 27 26 25
117172023-09-22T23:29:11.344ZWrote out file "/tmp/test_repair_verify.data"
117182023-09-22T23:29:11.347Z
117192023-09-22T23:29:11.350ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
117202023-09-22T23:29:11.353Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:11.088919718Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1524}
117212023-09-22T23:29:11.356Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:11.089264953Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1524}
117222023-09-22T23:29:11.359Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:11.089288536Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1524}
117232023-09-22T23:29:11.362ZError: Difference in extent metadata found!
117242023-09-22T23:29:11.365Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:11.113221245Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1524}
117252023-09-22T23:29:11.368Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:11.113254318Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1524}
117262023-09-22T23:29:11.371Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:11.113273196Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1524}
117272023-09-22T23:29:11.374Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:11.130386344Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1524}
117282023-09-22T23:29:11.377Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:11.130416589Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1524}
117292023-09-22T23:29:11.380Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:11.130435346Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1524}
117302023-09-22T23:29:11.384ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
117312023-09-22T23:29:11.387Z 0 000-019 80 80 80 123 123 123 F F F
117322023-09-22T23:29:11.390Z 1 020-039 88 82 88 134 125 134 F F F <---
117332023-09-22T23:29:11.393Z 2 040-059 88 83 88 134 126 134 T F T <---
117342023-09-22T23:29:11.396Z 3 060-079 86 86 86 129 129 129 F F F
117352023-09-22T23:29:11.399Z 4 080-099 86 86 86 130 130 130 F F F
117362023-09-22T23:29:11.401Z 5 100-119 86 86 86 130 130 130 F F F
117372023-09-22T23:29:11.404Z 6 120-139 85 85 85 128 128 128 F F F
117382023-09-22T23:29:11.408Z 7 140-159 88 88 88 132 132 132 F F F
117392023-09-22T23:29:11.411Z 8 160-179 86 86 86 130 130 130 F F F
117402023-09-22T23:29:11.414Z 9 180-199 88 85 88 134 128 134 F F F <---
117412023-09-22T23:29:11.417Z 10 200-219 88 86 88 134 130 134 F F F <---
117422023-09-22T23:29:11.420Z 11 220-239 88 86 88 135 130 135 F F F <---
117432023-09-22T23:29:11.423Z 12 240-259 88 87 88 135 131 135 T F T <---
117442023-09-22T23:29:11.426Z 13 260-279 86 86 86 130 130 130 T F T <---
117452023-09-22T23:29:11.430Z 14 280-299 86 86 86 130 130 130 F F F
117462023-09-22T23:29:11.433Z 15 300-319 83 83 83 126 126 126 F F F
117472023-09-22T23:29:11.437Z 16 320-339 87 87 87 131 131 131 F F F
117482023-09-22T23:29:11.440Z 17 340-359 88 86 88 135 130 135 F F F <---
117492023-09-22T23:29:11.443Z 18 360-379 88 78 88 135 121 135 F F F <---
117502023-09-22T23:29:11.445Z 19 380-399 88 85 88 134 128 134 F F F <---
117512023-09-22T23:29:11.448Z 20 400-419 83 83 83 126 126 126 F F F
117522023-09-22T23:29:11.451Z 21 420-439 88 85 88 134 128 134 F F F <---
117532023-09-22T23:29:11.454Z 22 440-459 88 85 88 134 128 134 F F F <---
117542023-09-22T23:29:11.457Z 23 460-479 88 80 88 134 123 134 F F F <---
117552023-09-22T23:29:11.460Z 24 480-499 87 87 87 131 131 131 F F F
117562023-09-22T23:29:11.465Z 25 500-519 78 78 78 119 119 119 F F F
117572023-09-22T23:29:11.468Z 26 520-539 84 84 84 127 127 127 F F F
117582023-09-22T23:29:11.471Z 27 540-559 85 85 85 128 128 128 F F F
117592023-09-22T23:29:11.475Z 28 560-579 86 86 86 130 130 130 F F F
117602023-09-22T23:29:11.478Z 29 580-599 88 86 88 134 130 134 F F F <---
117612023-09-22T23:29:11.481ZMax gen: 88, Max flush: 135
117622023-09-22T23:29:11.484ZA Difference in extent metadata is expected here
117632023-09-22T23:29:11.487ZOn loop 44
117642023-09-22T23:29:11.491Z
117652023-09-22T23:29:11.494ZVerifying data now
117662023-09-22T23:29:17.622ZNo change after verify
117672023-09-22T23:29:17.625ZLoop: 44 Downstairs dump after verify (and repair):
117682023-09-22T23:29:17.641Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.640804697Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1529}
117692023-09-22T23:29:17.644Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.641132088Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1529}
117702023-09-22T23:29:17.646Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.641160085Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1529}
117712023-09-22T23:29:17.660Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.659693118Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1529}
117722023-09-22T23:29:17.664Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.659744631Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1529}
117732023-09-22T23:29:17.667Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.659765209Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1529}
117742023-09-22T23:29:17.670Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.670635326Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1529}
117752023-09-22T23:29:17.673Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.670686923Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1529}
117762023-09-22T23:29:17.677Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.670707302Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1529}
117772023-09-22T23:29:17.681ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
117782023-09-22T23:29:17.684Z 0 000-019 80 80 80 123 123 123 F F F
117792023-09-22T23:29:17.687Z 1 020-039 88 88 88 134 134 134 F F F
117802023-09-22T23:29:17.690Z 2 040-059 89 89 89 136 136 136 F F F
117812023-09-22T23:29:17.693Z 3 060-079 86 86 86 129 129 129 F F F
117822023-09-22T23:29:17.696Z 4 080-099 86 86 86 130 130 130 F F F
117832023-09-22T23:29:17.699Z 5 100-119 86 86 86 130 130 130 F F F
117842023-09-22T23:29:17.702Z 6 120-139 85 85 85 128 128 128 F F F
117852023-09-22T23:29:17.705Z 7 140-159 88 88 88 132 132 132 F F F
117862023-09-22T23:29:17.708Z 8 160-179 86 86 86 130 130 130 F F F
117872023-09-22T23:29:17.711Z 9 180-199 88 88 88 134 134 134 F F F
117882023-09-22T23:29:17.713Z 10 200-219 88 88 88 134 134 134 F F F
117892023-09-22T23:29:17.717Z 11 220-239 88 88 88 135 135 135 F F F
117902023-09-22T23:29:17.720Z 12 240-259 89 89 89 136 136 136 F F F
117912023-09-22T23:29:17.723Z 13 260-279 89 89 89 136 136 136 F F F
117922023-09-22T23:29:17.726Z 14 280-299 86 86 86 130 130 130 F F F
117932023-09-22T23:29:17.729Z 15 300-319 83 83 83 126 126 126 F F F
117942023-09-22T23:29:17.732Z 16 320-339 87 87 87 131 131 131 F F F
117952023-09-22T23:29:17.735Z 17 340-359 88 88 88 135 135 135 F F F
117962023-09-22T23:29:17.738Z 18 360-379 88 88 88 135 135 135 F F F
117972023-09-22T23:29:17.741Z 19 380-399 88 88 88 134 134 134 F F F
117982023-09-22T23:29:17.744Z 20 400-419 83 83 83 126 126 126 F F F
117992023-09-22T23:29:17.747Z 21 420-439 88 88 88 134 134 134 F F F
118002023-09-22T23:29:17.750Z 22 440-459 88 88 88 134 134 134 F F F
118012023-09-22T23:29:17.753Z 23 460-479 88 88 88 134 134 134 F F F
118022023-09-22T23:29:17.756Z 24 480-499 87 87 87 131 131 131 F F F
118032023-09-22T23:29:17.759Z 25 500-519 78 78 78 119 119 119 F F F
118042023-09-22T23:29:17.762Z 26 520-539 84 84 84 127 127 127 F F F
118052023-09-22T23:29:17.765Z 27 540-559 85 85 85 128 128 128 F F F
118062023-09-22T23:29:17.768Z 28 560-579 86 86 86 130 130 130 F F F
118072023-09-22T23:29:17.771Z 29 580-599 88 88 88 134 134 134 F F F
118082023-09-22T23:29:17.774ZMax gen: 89, Max flush: 136
118092023-09-22T23:29:17.777Z
118102023-09-22T23:29:17.779ZBegin loop 45
118112023-09-22T23:29:17.782ZDownstairs to restart: 1
118122023-09-22T23:29:17.785Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.717707152Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118132023-09-22T23:29:17.788Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.718075183Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118142023-09-22T23:29:17.791Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.71812795Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118152023-09-22T23:29:17.794Z{"msg":"Crucible stats registered with UUID: 492be8a0-ecb0-47a9-880c-8711e4065484","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.718171989Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118162023-09-22T23:29:17.798Z{"msg":"Crucible 492be8a0-ecb0-47a9-880c-8711e4065484 has session id: 57e7ed09-3b03-4a3f-b15b-bde00465931e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.718223689Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118172023-09-22T23:29:17.801ZCrucible runtime is spawned
118182023-09-22T23:29:17.804ZThe guest has requested activation with gen:90
118192023-09-22T23:29:17.807Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.718473657Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531,"task":"up_listen"}
118202023-09-22T23:29:17.811Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.718551584Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118212023-09-22T23:29:17.817Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.718618872Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118222023-09-22T23:29:17.819Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.718686158Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531,"looper":"2"}
118232023-09-22T23:29:17.823Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.718973036Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531,"looper":"0"}
118242023-09-22T23:29:17.826Z{"msg":"[2] 492be8a0-ecb0-47a9-880c-8711e4065484 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.71907948Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531,"looper":"2"}
118252023-09-22T23:29:17.829Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.719122273Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531,"looper":"1"}
118262023-09-22T23:29:17.832Z{"msg":"[0] 492be8a0-ecb0-47a9-880c-8711e4065484 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.719193198Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531,"looper":"0"}
118272023-09-22T23:29:17.835Z{"msg":"492be8a0-ecb0-47a9-880c-8711e4065484 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.719519704Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118282023-09-22T23:29:17.838Z{"msg":"Set desired generation to :90","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.719570562Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118292023-09-22T23:29:17.841Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.719618264Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118302023-09-22T23:29:17.844Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.719947351Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118312023-09-22T23:29:17.847Z{"msg":"[2] 492be8a0-ecb0-47a9-880c-8711e4065484 (eefee60c-17f4-4a51-a5c7-a1e8b8a95952) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.7202995Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118322023-09-22T23:29:17.850Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.720394336Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118332023-09-22T23:29:17.853Z{"msg":"[2] client is_active_req TRUE, promote! session eefee60c-17f4-4a51-a5c7-a1e8b8a95952","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.720467555Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118342023-09-22T23:29:17.856Z{"msg":"[0] 492be8a0-ecb0-47a9-880c-8711e4065484 (eefee60c-17f4-4a51-a5c7-a1e8b8a95952) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.720558219Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118352023-09-22T23:29:17.859Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.72062436Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118362023-09-22T23:29:17.862Z{"msg":"[0] client is_active_req TRUE, promote! session eefee60c-17f4-4a51-a5c7-a1e8b8a95952","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.720705669Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118372023-09-22T23:29:17.865Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.72098751Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118382023-09-22T23:29:17.868Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.721062224Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118392023-09-22T23:29:17.870Z{"msg":"492be8a0-ecb0-47a9-880c-8711e4065484 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.721117732Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118402023-09-22T23:29:17.873Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.721239682Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118412023-09-22T23:29:17.876Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.721286754Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118422023-09-22T23:29:17.879Z{"msg":"492be8a0-ecb0-47a9-880c-8711e4065484 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.721331026Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118432023-09-22T23:29:17.882Z{"msg":"[2] 492be8a0-ecb0-47a9-880c-8711e4065484 (eefee60c-17f4-4a51-a5c7-a1e8b8a95952) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.723350844Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118442023-09-22T23:29:17.886Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.723408541Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118452023-09-22T23:29:17.889Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:29:17.723515475Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118462023-09-22T23:29:17.891Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.72358301Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118472023-09-22T23:29:17.895Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.723628188Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118482023-09-22T23:29:17.898Z{"msg":"492be8a0-ecb0-47a9-880c-8711e4065484 WaitActive New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.723668061Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118492023-09-22T23:29:17.901Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.723707203Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118502023-09-22T23:29:17.904Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.72374561Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118512023-09-22T23:29:17.907Z{"msg":"[0] 492be8a0-ecb0-47a9-880c-8711e4065484 (eefee60c-17f4-4a51-a5c7-a1e8b8a95952) WaitActive New WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.723786564Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118522023-09-22T23:29:17.910Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.723825515Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118532023-09-22T23:29:17.922Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:29:17.723863577Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118542023-09-22T23:29:17.941Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.723901109Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118552023-09-22T23:29:17.950Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.723939151Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118562023-09-22T23:29:17.953Z{"msg":"492be8a0-ecb0-47a9-880c-8711e4065484 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.723977481Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118572023-09-22T23:29:17.961Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.724015568Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118582023-09-22T23:29:17.966Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:17.724034747Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118592023-09-22T23:29:18.721Z{"msg":"[1] 492be8a0-ecb0-47a9-880c-8711e4065484 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.720807852Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531,"looper":"1"}
118602023-09-22T23:29:18.724Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.720889312Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118612023-09-22T23:29:18.727Z{"msg":"[1] 492be8a0-ecb0-47a9-880c-8711e4065484 (eefee60c-17f4-4a51-a5c7-a1e8b8a95952) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.722090096Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118622023-09-22T23:29:18.730Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.722141049Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118632023-09-22T23:29:18.733Z{"msg":"[1] client is_active_req TRUE, promote! session eefee60c-17f4-4a51-a5c7-a1e8b8a95952","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.72218455Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118642023-09-22T23:29:18.736Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.722784502Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118652023-09-22T23:29:18.739Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.722840931Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118662023-09-22T23:29:18.743Z{"msg":"492be8a0-ecb0-47a9-880c-8711e4065484 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.72288278Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118672023-09-22T23:29:18.746Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.724042956Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118682023-09-22T23:29:18.749Z{"msg":"[1] 492be8a0-ecb0-47a9-880c-8711e4065484 (eefee60c-17f4-4a51-a5c7-a1e8b8a95952) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.725344129Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118692023-09-22T23:29:18.752Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.725399687Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118702023-09-22T23:29:18.755Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:29:18.725440695Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118712023-09-22T23:29:18.758Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.725480278Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118722023-09-22T23:29:18.762Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.725518935Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118732023-09-22T23:29:18.765Z{"msg":"492be8a0-ecb0-47a9-880c-8711e4065484 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.725558101Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118742023-09-22T23:29:18.767Z{"msg":"[0]R flush_numbers[0..12]: [123, 134, 136, 129, 130, 130, 128, 132, 130, 134, 134, 135]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.725598137Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118752023-09-22T23:29:18.770Z{"msg":"[0]R generation[0..12]: [80, 88, 89, 86, 86, 86, 85, 88, 86, 88, 88, 88]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.7256373Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118762023-09-22T23:29:18.773Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.725677374Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118772023-09-22T23:29:18.776Z{"msg":"[1]R flush_numbers[0..12]: [123, 134, 136, 129, 130, 130, 128, 132, 130, 134, 134, 135]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.725722208Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118782023-09-22T23:29:18.779Z{"msg":"[1]R generation[0..12]: [80, 88, 89, 86, 86, 86, 85, 88, 86, 88, 88, 88]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:90
118792023-09-22T23:29:18.782ZWait for a query_work_queue command to finish before sending IO
118802023-09-22T23:29:18.785Z,"time":"2023-09-22T23:29:18.725761076Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118812023-09-22T23:29:18.788Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.725934175Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118822023-09-22T23:29:18.792Z{"msg":"[2]R flush_numbers[0..12]: [123, 134, 136, 129, 130, 130, 128, 132, 130, 134, 134, 135]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.72606332Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118832023-09-22T23:29:18.795Z{"msg":"[2]R generation[0..12]: [80, 88, 89, 86, 86, 86, 85, 88, 86, 88, 88, 88]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.726112411Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118842023-09-22T23:29:18.801Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.726153129Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118852023-09-22T23:29:18.804Z{"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.726193806Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118862023-09-22T23:29:18.806Z{"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.726240187Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118872023-09-22T23:29:18.809Z{"msg":"Next flush: 137","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.726277641Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118882023-09-22T23:29:18.812Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.726317354Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118892023-09-22T23:29:18.815Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.726366078Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118902023-09-22T23:29:18.818Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.72640465Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118912023-09-22T23:29:18.822Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.726444122Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118922023-09-22T23:29:18.825Z{"msg":"492be8a0-ecb0-47a9-880c-8711e4065484 is now active with session: eefee60c-17f4-4a51-a5c7-a1e8b8a95952","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.72646691Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118932023-09-22T23:29:18.828Z{"msg":"492be8a0-ecb0-47a9-880c-8711e4065484 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.726506526Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118942023-09-22T23:29:18.831Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.726544989Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118952023-09-22T23:29:18.834Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.726583026Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118962023-09-22T23:29:18.837Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.726621194Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118972023-09-22T23:29:18.840Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.726668658Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118982023-09-22T23:29:18.843Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.726706184Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
118992023-09-22T23:29:18.846Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.726743964Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
119002023-09-22T23:29:18.849Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
119012023-09-22T23:29:18.852Z,"time":"2023-09-22T23:29:18.726781886Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
119022023-09-22T23:29:18.863Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.726802431Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
119032023-09-22T23:29:18.866Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.72683841Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
119042023-09-22T23:29:18.870Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.726876298Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
119052023-09-22T23:29:18.872Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.726914046Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1531}
119062023-09-22T23:29:18.875ZLoading write count information from file "/tmp/test_repair_verify.data"
119072023-09-22T23:29:18.878ZRun Repair workload
119082023-09-22T23:29:18.881Z01/30 Write block 332 len 4096 data: 7
119092023-09-22T23:29:18.884Z02/30 Read block 434 len 20480
119102023-09-22T23:29:18.888Z03/30 Read block 575 len 12288
119112023-09-22T23:29:18.890Z04/30 Write block 460 len 8192 data: 9 10
119122023-09-22T23:29:18.894Z05/30 Write block 108 len 28672 data: 10 10 11 8 7 7 5
119132023-09-22T23:29:18.897Z06/30 Read block 514 len 16384
119142023-09-22T23:29:18.905Z07/30 Read block 580 len 28672
119152023-09-22T23:29:18.909Z08/30 Read block 532 len 4096
119162023-09-22T23:29:18.914Z09/30 Write block 436 len 32768 data: 8 7 6 4 4 6 7 9
119172023-09-22T23:29:18.918Z10/30 Read block 41 len 36864
119182023-09-22T23:29:18.937Z11/30 Read block 360 len 40960
119192023-09-22T23:29:18.942Z12/30 Write block 395 len 16384 data: 7 7 7 8
119202023-09-22T23:29:18.946Z13/30 Read block 245 len 8192
119212023-09-22T23:29:18.950Z14/30 Write block 342 len 28672 data: 7 6 7 7 8 8 9
119222023-09-22T23:29:18.953Z15/30 Write block 91 len 20480 data: 8 9 8 8 10
119232023-09-22T23:29:18.957Z16/30 Write block 465 len 4096 data: 12
119242023-09-22T23:29:18.960Z17/30 Read block 83 len 28672
119252023-09-22T23:29:18.966Z18/30 Read block 488 len 20480
119262023-09-22T23:29:18.973Z19/30 Write block 62 len 24576 data: 13 16 15 17 16 16
119272023-09-22T23:29:18.980Z20/30 Write block 55 len 4096 data: 4
119282023-09-22T23:29:18.987Z21/30 Write block 190 len 28672 data: 8 8 7 7 8 9 9
119292023-09-22T23:29:19.403Z22/30 Write block 509 len 12288 data: 8 7 6
119302023-09-22T23:29:19.414Z23/30 Write block 385 len 16384 data: 6 5 7 7
119312023-09-22T23:29:19.417Z24/30 Write block 78 len 20480 data: 10 13 11 13 12
119322023-09-22T23:29:19.421Z25/30 Write block 488 len 32768 data: 12 13 12 13 12 12 11 13
119332023-09-22T23:29:19.424Z26/30 Write block 294 len 24576 data: 7 8 7 5 6 7
119342023-09-22T23:29:19.427Z27/30 Read block 57 len 16384
119352023-09-22T23:29:19.430Z28/30 Read block 573 len 20480
119362023-09-22T23:29:19.433Z29/30 Read block 72 len 32768
119372023-09-22T23:29:19.436Z30/30 Write block 364 len 24576 data: 5 5 6 7 8 6
119382023-09-22T23:29:19.439Z----------------------------------------------------------------
119392023-09-22T23:29:19.442Z Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:30
119402023-09-22T23:29:19.445ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
119412023-09-22T23:29:19.447Z 1 Acked 1000 Write 1 Done Sent Done false
119422023-09-22T23:29:19.451Z 2 Acked 1001 Read 5 Done Sent Done false
119432023-09-22T23:29:19.457Z 3 Acked 1002 Read 3 Done Sent Done false
119442023-09-22T23:29:19.461Z 4 Acked 1003 Write 2 Done Sent Done false
119452023-09-22T23:29:19.464Z 5 Acked 1004 Write 7 Done Sent Done false
119462023-09-22T23:29:19.467Z 6 Acked 1005 Read 4 Done Sent Done false
119472023-09-22T23:29:19.470Z 7 Acked 1006 Read 7 Done Sent Done false
119482023-09-22T23:29:19.474Z 8 Acked 1007 Read 1 Done Sent Done false
119492023-09-22T23:29:19.477Z 9 Acked 1008 Write 8 Done Sent Done false
119502023-09-22T23:29:19.480Z 10 Acked 1009 Read 9 Done Sent Done false
119512023-09-22T23:29:19.483Z 11 Acked 1010 Read 10 Done Sent Done false
119522023-09-22T23:29:19.486Z 12 Acked 1011 Write 4 Done Sent Done false
119532023-09-22T23:29:19.489Z 13 Acked 1012 Read 2 Done Sent Done false
119542023-09-22T23:29:19.492Z 14 Acked 1013 Write 7 Done Sent Done false
119552023-09-22T23:29:19.495Z 15 Acked 1014 Write 5 Done Sent Done false
119562023-09-22T23:29:19.500Z 16 Acked 1015 Write 1 Done Sent Done false
119572023-09-22T23:29:19.504Z 17 Acked 1016 Read 7 Done Sent Done false
119582023-09-22T23:29:19.507Z 18 Acked 1017 Read 5 Done Sent Done false
119592023-09-22T23:29:19.537Z 19 Acked 1018 Write 6 Done Sent Done false
119602023-09-22T23:29:19.541Z 20 Acked 1019 Write 1 Done Sent Done false
119612023-09-22T23:29:19.544Z 21 Acked 1020 Write 7 Done Sent Done false
119622023-09-22T23:29:19.547Z 22 Acked 1021 Write 3 Done Sent Done false
119632023-09-22T23:29:19.550Z 23 Acked 1022 Write 4 Done Sent Done false
119642023-09-22T23:29:19.554Z 24 Acked 1023 Write 5 Done Sent Done false
119652023-09-22T23:29:19.557Z 25 Acked 1024 Write 8 Done Sent Done false
119662023-09-22T23:29:19.560Z 26 Acked 1025 Write 6 Done Sent Done false
119672023-09-22T23:29:19.564Z 27 Acked 1026 Read 4 Done Sent Done false
119682023-09-22T23:29:19.567Z 28 Acked 1027 Read 5 Done Sent Done false
119692023-09-22T23:29:19.570Z 29 Acked 1028 Read 8 Done Sent Done false
119702023-09-22T23:29:19.574Z 30 Acked 1029 Write 6 Sent Sent Sent false
119712023-09-22T23:29:19.577Z STATES DS:0 DS:1 DS:2 TOTAL
119722023-09-22T23:29:19.581Z New 0 0 0 0
119732023-09-22T23:29:19.585Z Sent 1 30 1 32
119742023-09-22T23:29:19.588Z Done 29 0 29 58
119752023-09-22T23:29:19.591Z Skipped 0 0 0 0
119762023-09-22T23:29:19.594Z Error 0 0 0 0
119772023-09-22T23:29:19.597ZLast Flush: 0 0 0
119782023-09-22T23:29:19.600ZDownstairs last five completed:
119792023-09-22T23:29:19.604ZUpstairs last five completed: 30 29 28 27 26 25
119802023-09-22T23:29:19.607ZWrote out file "/tmp/test_repair_verify.data"
119812023-09-22T23:29:19.613Z
119822023-09-22T23:29:19.617ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
119832023-09-22T23:29:19.620Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.852275085Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1532}
119842023-09-22T23:29:19.623ZError: Difference in extent metadata found!
119852023-09-22T23:29:19.625Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.852550344Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1532}
119862023-09-22T23:29:19.628Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.852573519Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1532}
119872023-09-22T23:29:19.632Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.89165502Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1532}
119882023-09-22T23:29:19.635Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.891691792Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1532}
119892023-09-22T23:29:19.638Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.891711953Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1532}
119902023-09-22T23:29:19.641Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.906718464Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1532}
119912023-09-22T23:29:19.644Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.906748704Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1532}
119922023-09-22T23:29:19.647Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:18.906767132Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1532}
119932023-09-22T23:29:19.650ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
119942023-09-22T23:29:19.654Z 0 000-019 80 80 80 123 123 123 F F F
119952023-09-22T23:29:19.657Z 1 020-039 88 88 88 134 134 134 F F F
119962023-09-22T23:29:19.660Z 2 040-059 89 89 89 136 136 136 T F T <---
119972023-09-22T23:29:19.663Z 3 060-079 86 86 86 129 129 129 T F T <---
119982023-09-22T23:29:19.666Z 4 080-099 86 86 86 130 130 130 T F T <---
119992023-09-22T23:29:19.669Z 5 100-119 86 86 86 130 130 130 T F T <---
120002023-09-22T23:29:19.672Z 6 120-139 85 85 85 128 128 128 F F F
120012023-09-22T23:29:19.676Z 7 140-159 88 88 88 132 132 132 F F F
120022023-09-22T23:29:19.678Z 8 160-179 86 86 86 130 130 130 F F F
120032023-09-22T23:29:19.682Z 9 180-199 88 88 88 134 134 134 T F T <---
120042023-09-22T23:29:19.685Z 10 200-219 88 88 88 134 134 134 F F F
120052023-09-22T23:29:19.688Z 11 220-239 88 88 88 135 135 135 F F F
120062023-09-22T23:29:19.691Z 12 240-259 89 89 89 136 136 136 F F F
120072023-09-22T23:29:19.694Z 13 260-279 89 89 89 136 136 136 F F F
120082023-09-22T23:29:19.697Z 14 280-299 86 86 86 130 130 130 T F T <---
120092023-09-22T23:29:19.701Z 15 300-319 83 83 83 126 126 126 F F F
120102023-09-22T23:29:19.704Z 16 320-339 87 87 87 131 131 131 T F T <---
120112023-09-22T23:29:19.707Z 17 340-359 88 88 88 135 135 135 T F T <---
120122023-09-22T23:29:19.710Z 18 360-379 88 88 88 135 135 135 T F T <---
120132023-09-22T23:29:19.713Z 19 380-399 88 88 88 134 134 134 T F T <---
120142023-09-22T23:29:19.717Z 20 400-419 83 83 83 126 126 126 F F F
120152023-09-22T23:29:19.721Z 21 420-439 88 88 88 134 134 134 T F T <---
120162023-09-22T23:29:19.724Z 22 440-459 88 88 88 134 134 134 T F T <---
120172023-09-22T23:29:19.727Z 23 460-479 88 88 88 134 134 134 T F T <---
120182023-09-22T23:29:19.730Z 24 480-499 87 87 87 131 131 131 T F T <---
120192023-09-22T23:29:19.733Z 25 500-519 78 78 78 119 119 119 T F T <---
120202023-09-22T23:29:19.736Z 26 520-539 84 84 84 127 127 127 F F F
120212023-09-22T23:29:19.739Z 27 540-559 85 85 85 128 128 128 F F F
120222023-09-22T23:29:19.742Z 28 560-579 86 86 86 130 130 130 F F F
120232023-09-22T23:29:19.745Z 29 580-599 88 88 88 134 134 134 F F F
120242023-09-22T23:29:19.748ZMax gen: 89, Max flush: 136
120252023-09-22T23:29:19.752ZA Difference in extent metadata is expected here
120262023-09-22T23:29:19.755ZOn loop 45
120272023-09-22T23:29:19.758Z
120282023-09-22T23:29:19.761ZVerifying data now
120292023-09-22T23:29:25.528ZNo change after verify
120302023-09-22T23:29:25.531ZLoop: 45 Downstairs dump after verify (and repair):
120312023-09-22T23:29:25.545Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.545143402Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1537}
120322023-09-22T23:29:25.549Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.545514211Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1537}
120332023-09-22T23:29:25.552Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.545539076Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1537}
120342023-09-22T23:29:25.566Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.564856166Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1537}
120352023-09-22T23:29:25.569Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.564919234Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1537}
120362023-09-22T23:29:25.572Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.564940039Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1537}
120372023-09-22T23:29:25.576Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.576186696Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1537}
120382023-09-22T23:29:25.579Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.576237972Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1537}
120392023-09-22T23:29:25.582Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.576258273Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1537}
120402023-09-22T23:29:25.588ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
120412023-09-22T23:29:25.591Z 0 000-019 80 80 80 123 123 123 F F F
120422023-09-22T23:29:25.594Z 1 020-039 88 88 88 134 134 134 F F F
120432023-09-22T23:29:25.598Z 2 040-059 90 90 90 137 137 137 F F F
120442023-09-22T23:29:25.602Z 3 060-079 90 90 90 137 137 137 F F F
120452023-09-22T23:29:25.606Z 4 080-099 90 90 90 137 137 137 F F F
120462023-09-22T23:29:25.609Z 5 100-119 90 90 90 137 137 137 F F F
120472023-09-22T23:29:25.612Z 6 120-139 85 85 85 128 128 128 F F F
120482023-09-22T23:29:25.615Z 7 140-159 88 88 88 132 132 132 F F F
120492023-09-22T23:29:25.619Z 8 160-179 86 86 86 130 130 130 F F F
120502023-09-22T23:29:25.623Z 9 180-199 90 90 90 137 137 137 F F F
120512023-09-22T23:29:25.626Z 10 200-219 88 88 88 134 134 134 F F F
120522023-09-22T23:29:25.629Z 11 220-239 88 88 88 135 135 135 F F F
120532023-09-22T23:29:25.632Z 12 240-259 89 89 89 136 136 136 F F F
120542023-09-22T23:29:25.635Z 13 260-279 89 89 89 136 136 136 F F F
120552023-09-22T23:29:25.638Z 14 280-299 90 90 90 137 137 137 F F F
120562023-09-22T23:29:25.641Z 15 300-319 83 83 83 126 126 126 F F F
120572023-09-22T23:29:25.644Z 16 320-339 90 90 90 137 137 137 F F F
120582023-09-22T23:29:25.647Z 17 340-359 90 90 90 137 137 137 F F F
120592023-09-22T23:29:25.672Z 18 360-379 90 90 90 137 137 137 F F F
120602023-09-22T23:29:25.677Z 19 380-399 90 90 90 137 137 137 F F F
120612023-09-22T23:29:25.680Z 20 400-419 83 83 83 126 126 126 F F F
120622023-09-22T23:29:25.684Z 21 420-439 90 90 90 137 137 137 F F F
120632023-09-22T23:29:25.687Z 22 440-459 90 90 90 137 137 137 F F F
120642023-09-22T23:29:25.689Z 23 460-479 90 90 90 137 137 137 F F F
120652023-09-22T23:29:25.693Z 24 480-499 90 90 90 137 137 137 F F F
120662023-09-22T23:29:25.696Z 25 500-519 90 90 90 137 137 137 F F F
120672023-09-22T23:29:25.699Z 26 520-539 84 84 84 127 127 127 F F F
120682023-09-22T23:29:25.702Z 27 540-559 85 85 85 128 128 128 F F F
120692023-09-22T23:29:25.705Z 28 560-579 86 86 86 130 130 130 F F F
120702023-09-22T23:29:25.708Z 29 580-599 88 88 88 134 134 134 F F F
120712023-09-22T23:29:25.712ZMax gen: 90, Max flush: 137
120722023-09-22T23:29:25.715Z
120732023-09-22T23:29:25.718ZBegin loop 46
120742023-09-22T23:29:25.721ZDownstairs to restart: 1
120752023-09-22T23:29:25.724Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.62677654Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
120762023-09-22T23:29:25.727Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.627152682Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
120772023-09-22T23:29:25.730Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.627222982Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
120782023-09-22T23:29:25.733Z{"msg":"Crucible stats registered with UUID: 86e37865-df9f-4f8b-ae0d-f5d0a32c1ad0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.627269087Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
120792023-09-22T23:29:25.736Z{"msg":"Crucible 86e37865-df9f-4f8b-ae0d-f5d0a32c1ad0 has session id: 9e369e8a-e446-48a7-925a-264342c1420e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.627312895Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
120802023-09-22T23:29:25.738ZCrucible runtime is spawned
120812023-09-22T23:29:25.741ZThe guest has requested activation with gen:92
120822023-09-22T23:29:25.744Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.62757479Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539,"task":"up_listen"}
120832023-09-22T23:29:25.747Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.627653358Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
120842023-09-22T23:29:25.751Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.627718835Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
120852023-09-22T23:29:25.754Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.627781423Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539,"looper":"2"}
120862023-09-22T23:29:25.757Z{"msg":"[2] 86e37865-df9f-4f8b-ae0d-f5d0a32c1ad0 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.627920792Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539,"looper":"2"}
120872023-09-22T23:29:25.760Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.628005236Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539,"looper":"0"}
120882023-09-22T23:29:25.763Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.628255306Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539,"looper":"1"}
120892023-09-22T23:29:25.766Z{"msg":"[0] 86e37865-df9f-4f8b-ae0d-f5d0a32c1ad0 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.628318183Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539,"looper":"0"}
120902023-09-22T23:29:25.769Z{"msg":"86e37865-df9f-4f8b-ae0d-f5d0a32c1ad0 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.628776397Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
120912023-09-22T23:29:25.772Z{"msg":"Set desired generation to :92","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.628830211Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
120922023-09-22T23:29:25.775Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.628870464Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
120932023-09-22T23:29:25.778Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.62923151Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
120942023-09-22T23:29:25.781Z{"msg":"[2] 86e37865-df9f-4f8b-ae0d-f5d0a32c1ad0 (001ffca8-2c50-4807-b47a-b8c2958082f2) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.629635546Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
120952023-09-22T23:29:25.784Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.629764834Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
120962023-09-22T23:29:25.787Z{"msg":"[2] client is_active_req TRUE, promote! session 001ffca8-2c50-4807-b47a-b8c2958082f2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.629822935Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
120972023-09-22T23:29:25.790Z{"msg":"[0] 86e37865-df9f-4f8b-ae0d-f5d0a32c1ad0 (001ffca8-2c50-4807-b47a-b8c2958082f2) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.629868971Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
120982023-09-22T23:29:25.793Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.62998089Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
120992023-09-22T23:29:25.796Z{"msg":"[0] client is_active_req TRUE, promote! session 001ffca8-2c50-4807-b47a-b8c2958082f2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.630050041Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121002023-09-22T23:29:25.799Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.630385986Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121012023-09-22T23:29:25.802Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.630451832Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121022023-09-22T23:29:25.805Z{"msg":"86e37865-df9f-4f8b-ae0d-f5d0a32c1ad0 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.630551973Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121032023-09-22T23:29:25.808Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.630694492Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121042023-09-22T23:29:25.812Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.630755286Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121052023-09-22T23:29:25.817Z{"msg":"86e37865-df9f-4f8b-ae0d-f5d0a32c1ad0 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.630788636Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121062023-09-22T23:29:25.820Z{"msg":"[2] 86e37865-df9f-4f8b-ae0d-f5d0a32c1ad0 (001ffca8-2c50-4807-b47a-b8c2958082f2) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.632980768Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121072023-09-22T23:29:25.827Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.633034496Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121082023-09-22T23:29:25.830Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:29:25.633078424Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121092023-09-22T23:29:25.836Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.633197667Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121102023-09-22T23:29:25.845Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.633253293Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121112023-09-22T23:29:25.851Z{"msg":"86e37865-df9f-4f8b-ae0d-f5d0a32c1ad0 WaitActive New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.633295352Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121122023-09-22T23:29:25.856Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.633333941Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121132023-09-22T23:29:25.860Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.633371481Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121142023-09-22T23:29:25.865Z{"msg":"[0] 86e37865-df9f-4f8b-ae0d-f5d0a32c1ad0 (001ffca8-2c50-4807-b47a-b8c2958082f2) WaitActive New WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.633412607Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121152023-09-22T23:29:25.868Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.633452331Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121162023-09-22T23:29:25.871Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:29:25.633490265Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121172023-09-22T23:29:25.874Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.63352798Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121182023-09-22T23:29:25.879Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.63356657Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121192023-09-22T23:29:25.886Z{"msg":"86e37865-df9f-4f8b-ae0d-f5d0a32c1ad0 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.633605015Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121202023-09-22T23:29:25.893Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.63362486Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121212023-09-22T23:29:25.899Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:25.633642338Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121222023-09-22T23:29:26.630Z{"msg":"[1] 86e37865-df9f-4f8b-ae0d-f5d0a32c1ad0 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.629683363Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539,"looper":"1"}
121232023-09-22T23:29:26.634Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.629765231Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121242023-09-22T23:29:26.637Z{"msg":"[1] 86e37865-df9f-4f8b-ae0d-f5d0a32c1ad0 (001ffca8-2c50-4807-b47a-b8c2958082f2) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.630843533Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121252023-09-22T23:29:26.641Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.630901919Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121262023-09-22T23:29:26.644Z{"msg":"[1] client is_active_req TRUE, promote! session 001ffca8-2c50-4807-b47a-b8c2958082f2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.630945279Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121272023-09-22T23:29:26.647Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.631534956Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121282023-09-22T23:29:26.652Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.63159228Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121292023-09-22T23:29:26.655Z{"msg":"86e37865-df9f-4f8b-ae0d-f5d0a32c1ad0 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.631635889Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121302023-09-22T23:29:26.661Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.633928995Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121312023-09-22T23:29:26.664Z{"msg":"[1] 86e37865-df9f-4f8b-ae0d-f5d0a32c1ad0 (001ffca8-2c50-4807-b47a-b8c2958082f2) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.634128488Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121322023-09-22T23:29:26.667Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.634198658Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121332023-09-22T23:29:26.693Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:29:26.634229723Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121342023-09-22T23:29:26.696Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.634290932Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121352023-09-22T23:29:26.699Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.634346638Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121362023-09-22T23:29:26.703Z{"msg":"86e37865-df9f-4f8b-ae0d-f5d0a32c1ad0 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.634403724Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121372023-09-22T23:29:26.706Z{"msg":"[0]R flush_numbers[0..12]: [123, 134, 137, 137, 137, 137, 128, 132, 130, 137, 134, 135]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.634466575Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121382023-09-22T23:29:26.710Z{"msg":"[0]R generation[0..12]: [80, 88, 90, 90, 90, 90, 85, 88, 86, 90, 88, 88]","v":0,"name":"crucible"The guest has finished waiting for activation with:92
121392023-09-22T23:29:26.713ZWait for a query_work_queue command to finish before sending IO
121402023-09-22T23:29:26.716Z,"level":30,"time":"2023-09-22T23:29:26.634689067Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121412023-09-22T23:29:26.719Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.634767599Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121422023-09-22T23:29:26.722Z{"msg":"[1]R flush_numbers[0..12]: [123, 134, 137, 137, 137, 137, 128, 132, 130, 137, 134, 135]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.634845721Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121432023-09-22T23:29:26.725Z{"msg":"[1]R generation[0..12]: [80, 88, 90, 90, 90, 90, 85, 88, 86, 90, 88, 88]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.634917967Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121442023-09-22T23:29:26.729Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.634981807Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121452023-09-22T23:29:26.735Z{"msg":"[2]R flush_numbers[0..12]: [123, 134, 137, 137, 137, 137, 128, 132, 130, 137, 134, 135]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.635040588Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121462023-09-22T23:29:26.742Z{"msg":"[2]R generation[0..12]: [80, 88, 90, 90, 90, 90, 85, 88, 86, 90, 88, 88]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.63508477Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121472023-09-22T23:29:26.746Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.635132587Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121482023-09-22T23:29:26.752Z{"msg":"Max found gen is 91","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.635177978Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121492023-09-22T23:29:26.760Z{"msg":"Generation requested: 92 >= found:91","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.635218253Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121502023-09-22T23:29:26.764Z{"msg":"Next flush: 138","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.635256493Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121512023-09-22T23:29:26.767Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.635299403Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121522023-09-22T23:29:26.770Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.635345825Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121532023-09-22T23:29:26.773Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.63538853Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121542023-09-22T23:29:26.776ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
121552023-09-22T23:29:26.779Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.635430766Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121562023-09-22T23:29:26.782Z{"msg":"86e37865-df9f-4f8b-ae0d-f5d0a32c1ad0 is now active with session: 001ffca8-2c50-4807-b47a-b8c2958082f2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.63547051Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121572023-09-22T23:29:26.786Z{"msg":"86e37865-df9f-4f8b-ae0d-f5d0a32c1ad0 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.635509323Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121582023-09-22T23:29:26.789Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.635547616Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121592023-09-22T23:29:26.792Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.635585249Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121602023-09-22T23:29:26.795Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.635623248Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121612023-09-22T23:29:26.798Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.635661189Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121622023-09-22T23:29:26.801Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.635698864Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121632023-09-22T23:29:26.805Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.635736721Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121642023-09-22T23:29:26.808Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.63577452Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121652023-09-22T23:29:26.811Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.635811745Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121662023-09-22T23:29:26.814Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.635849288Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121672023-09-22T23:29:26.817Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.635886852Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121682023-09-22T23:29:26.820Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.635924052Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1539}
121692023-09-22T23:29:26.824ZLoading write count information from file "/tmp/test_repair_verify.data"
121702023-09-22T23:29:26.827ZRun Repair workload
121712023-09-22T23:29:26.830Z01/30 Write block 265 len 32768 data: 9 9 12 9 10 10 11 11
121722023-09-22T23:29:26.833Z02/30 Read block 306 len 40960
121732023-09-22T23:29:26.836Z03/30 Write block 184 len 24576 data: 8 5 6 5 5 6
121742023-09-22T23:29:26.839Z04/30 Read block 50 len 28672
121752023-09-22T23:29:26.844Z05/30 Write block 428 len 24576 data: 8 9 10 9 11 10
121762023-09-22T23:29:26.847Z06/30 Read block 455 len 20480
121772023-09-22T23:29:26.850Z07/30 Write block 338 len 40960 data: 6 6 5 4 8 7 8 8 9 9
121782023-09-22T23:29:26.853Z08/30 Write block 239 len 12288 data: 8 8 9
121792023-09-22T23:29:26.856Z09/30 Write block 580 len 12288 data: 8 7 7
121802023-09-22T23:29:26.860Z10/30 Flush
121812023-09-22T23:29:26.863Z11/30 Write block 210 len 12288 data: 13 14 10
121822023-09-22T23:29:26.866Z12/30 Write block 509 len 8192 data: 9 8
121832023-09-22T23:29:26.869Z13/30 Read block 384 len 36864
121842023-09-22T23:29:26.873Z14/30 Write block 235 len 28672 data: 4 5 6 7 9 9 10
121852023-09-22T23:29:26.876Z15/30 Read block 226 len 8192
121862023-09-22T23:29:26.883Z16/30 Read block 97 len 8192
121872023-09-22T23:29:26.886Z17/30 Write block 102 len 32768 data: 8 9 11 13 12 11 11 11
121882023-09-22T23:29:26.890Z18/30 Read block 48 len 4096
121892023-09-22T23:29:26.893Z19/30 Write block 234 len 4096 data: 3
121902023-09-22T23:29:26.897Z20/30 Read block 181 len 36864
121912023-09-22T23:29:26.899Z21/30 Write block 35 len 32768 data: 7 10 11 11 10 10 9 8
121922023-09-22T23:29:26.903Z22/30 Read block 89 len 8192
121932023-09-22T23:29:26.906Z23/30 Read block 197 len 16384
121942023-09-22T23:29:26.910Z24/30 Read block 217 len 16384
121952023-09-22T23:29:27.321Z25/30 Write block 400 len 20480 data: 9 8 10 9 9
121962023-09-22T23:29:27.325Z26/30 Read block 437 len 40960
121972023-09-22T23:29:27.328Z27/30 Write block 394 len 36864 data: 6 8 8 8 9 7 10 9 11
121982023-09-22T23:29:27.331Z28/30 Write block 268 len 32768 data: 10 11 11 12 12 10 10 12
121992023-09-22T23:29:27.334Z29/30 Write block 299 len 8192 data: 8 6
122002023-09-22T23:29:27.337Z30/30 Write block 440 len 28672 data: 5 7 8 10 10 10 8
122012023-09-22T23:29:27.340Z----------------------------------------------------------------
122022023-09-22T23:29:27.343Z Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:20
122032023-09-22T23:29:27.346ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
122042023-09-22T23:29:27.350Z 11 Acked 1010 Write 3 Done Sent Done false
122052023-09-22T23:29:27.354Z 12 Acked 1011 Write 2 Done Sent Done false
122062023-09-22T23:29:27.357Z 13 Acked 1012 Read 9 Done Sent Done false
122072023-09-22T23:29:27.360Z 14 Acked 1013 Write 7 Done Sent Done false
122082023-09-22T23:29:27.363Z 15 Acked 1014 Read 2 Done Sent Done false
122092023-09-22T23:29:27.367Z 16 Acked 1015 Read 2 Done Sent Done false
122102023-09-22T23:29:27.371Z 17 Acked 1016 Write 8 Done Sent Done false
122112023-09-22T23:29:27.374Z 18 Acked 1017 Read 1 Done Sent Done false
122122023-09-22T23:29:27.376Z 19 Acked 1018 Write 1 Done Sent Done false
122132023-09-22T23:29:27.380Z 20 Acked 1019 Read 9 Done Sent Done false
122142023-09-22T23:29:27.383Z 21 Acked 1020 Write 8 Done Sent Done false
122152023-09-22T23:29:27.386Z 22 Acked 1021 Read 2 Done Sent Done false
122162023-09-22T23:29:27.389Z 23 Acked 1022 Read 4 Done Sent Done false
122172023-09-22T23:29:27.392Z 24 Acked 1023 Read 4 Done Sent Done false
122182023-09-22T23:29:27.395Z 25 Acked 1024 Write 5 Done Sent Done false
122192023-09-22T23:29:27.398Z 26 Acked 1025 Read 10 Done Sent Done false
122202023-09-22T23:29:27.401Z 27 Acked 1026 Write 9 Sent Sent Sent false
122212023-09-22T23:29:27.404Z 28 Acked 1027 Write 8 Sent Sent Sent false
122222023-09-22T23:29:27.407Z 29 Acked 1028 Write 2 Sent Sent Sent false
122232023-09-22T23:29:27.410Z 30 Acked 1029 Write 7 Sent Sent Sent false
122242023-09-22T23:29:27.414Z STATES DS:0 DS:1 DS:2 TOTAL
122252023-09-22T23:29:27.417Z New 0 0 0 0
122262023-09-22T23:29:27.420Z Sent 4 20 4 28
122272023-09-22T23:29:27.423Z Done 16 0 16 32
122282023-09-22T23:29:27.426Z Skipped 0 0 0 0
122292023-09-22T23:29:27.429Z Error 0 0 0 0
122302023-09-22T23:29:27.433ZLast Flush: 1009 1009 1009
122312023-09-22T23:29:27.436ZDownstairs last five completed: 1009 1008 1007 1006 1005 1004
122322023-09-22T23:29:27.438ZUpstairs last five completed: 30 29 28 27 26 25
122332023-09-22T23:29:27.443ZWrote out file "/tmp/test_repair_verify.data"
122342023-09-22T23:29:27.446Z
122352023-09-22T23:29:27.449ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
122362023-09-22T23:29:27.452Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.773547175Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1540}
122372023-09-22T23:29:27.455Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.773800313Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1540}
122382023-09-22T23:29:27.458Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.773823394Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1540}
122392023-09-22T23:29:27.461Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.805190621Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1540}
122402023-09-22T23:29:27.464Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.805225489Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1540}
122412023-09-22T23:29:27.467Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.805245152Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1540}
122422023-09-22T23:29:27.470Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.836601449Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1540}
122432023-09-22T23:29:27.474Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.83663539Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1540}
122442023-09-22T23:29:27.477Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:26.836654788Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1540}
122452023-09-22T23:29:27.480ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
122462023-09-22T23:29:27.484Z 0 000-019 80 80 80 123 123 123 F F F
122472023-09-22T23:29:27.490ZError: Difference in extent metadata found!
122482023-09-22T23:29:27.493Z 1 020-039 88 88 88 134 134 134 T F T <---
122492023-09-22T23:29:27.496Z 2 040-059 90 90 90 137 137 137 T F T <---
122502023-09-22T23:29:27.499Z 3 060-079 90 90 90 137 137 137 F F F
122512023-09-22T23:29:27.504Z 4 080-099 90 90 90 137 137 137 F F F
122522023-09-22T23:29:27.507Z 5 100-119 90 90 90 137 137 137 T F T <---
122532023-09-22T23:29:27.510Z 6 120-139 85 85 85 128 128 128 F F F
122542023-09-22T23:29:27.515Z 7 140-159 88 88 88 132 132 132 F F F
122552023-09-22T23:29:27.519Z 8 160-179 86 86 86 130 130 130 F F F
122562023-09-22T23:29:27.522Z 9 180-199 92 92 92 138 138 138 F F F
122572023-09-22T23:29:27.525Z 10 200-219 88 88 88 134 134 134 T F T <---
122582023-09-22T23:29:27.528Z 11 220-239 92 92 92 138 138 138 T F T <---
122592023-09-22T23:29:27.531Z 12 240-259 92 92 92 138 138 138 T F T <---
122602023-09-22T23:29:27.535Z 13 260-279 92 92 92 138 138 138 F F T <---
122612023-09-22T23:29:27.538Z 14 280-299 90 90 90 137 137 137 F F F
122622023-09-22T23:29:27.546Z 15 300-319 83 83 83 126 126 126 F F F
122632023-09-22T23:29:27.551Z 16 320-339 92 92 92 138 138 138 F F F
122642023-09-22T23:29:27.554Z 17 340-359 92 92 92 138 138 138 F F F
122652023-09-22T23:29:27.560Z 18 360-379 90 90 90 137 137 137 F F F
122662023-09-22T23:29:27.563Z 19 380-399 90 90 90 137 137 137 T F T <---
122672023-09-22T23:29:27.566Z 20 400-419 83 83 83 126 126 126 T F T <---
122682023-09-22T23:29:27.572Z 21 420-439 92 92 92 138 138 138 F F F
122692023-09-22T23:29:27.576Z 22 440-459 90 90 90 137 137 137 F F F
122702023-09-22T23:29:27.579Z 23 460-479 90 90 90 137 137 137 F F F
122712023-09-22T23:29:27.584Z 24 480-499 90 90 90 137 137 137 F F F
122722023-09-22T23:29:27.589Z 25 500-519 90 90 90 137 137 137 T F T <---
122732023-09-22T23:29:27.592Z 26 520-539 84 84 84 127 127 127 F F F
122742023-09-22T23:29:27.598Z 27 540-559 85 85 85 128 128 128 F F F
122752023-09-22T23:29:27.602Z 28 560-579 86 86 86 130 130 130 F F F
122762023-09-22T23:29:27.605Z 29 580-599 92 92 92 138 138 138 F F F
122772023-09-22T23:29:27.611ZMax gen: 92, Max flush: 138
122782023-09-22T23:29:27.616ZA Difference in extent metadata is expected here
122792023-09-22T23:29:27.619ZOn loop 46
122802023-09-22T23:29:27.626Z
122812023-09-22T23:29:27.630ZVerifying data now
122822023-09-22T23:29:33.154ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
122832023-09-22T23:29:33.157Z302,303c302
122842023-09-22T23:29:33.161Z< 7,
122852023-09-22T23:29:33.164Z< 5,
122862023-09-22T23:29:33.166Z---
122872023-09-22T23:29:33.169Z> 8,
122882023-09-22T23:29:33.172Z304a304
122892023-09-22T23:29:33.175Z> 6,
122902023-09-22T23:29:33.177Z443,444c443
122912023-09-22T23:29:33.181Z< 4,
122922023-09-22T23:29:33.183Z< 6,
122932023-09-22T23:29:33.186Z---
122942023-09-22T23:29:33.189Z> 5,
122952023-09-22T23:29:33.192Z446,449c445,449
122962023-09-22T23:29:33.195Z< 9,
122972023-09-22T23:29:33.203Z< 9,
122982023-09-22T23:29:33.206Z< 9,
122992023-09-22T23:29:33.210Z< 7,
123002023-09-22T23:29:33.213Z---
123012023-09-22T23:29:33.216Z> 8,
123022023-09-22T23:29:33.218Z> 10,
123032023-09-22T23:29:33.221Z> 10,
123042023-09-22T23:29:33.224Z> 10,
123052023-09-22T23:29:33.227Z> 8,
123062023-09-22T23:29:33.230Zdiff found after verify
123072023-09-22T23:29:33.233ZLoop: 46 Downstairs dump after verify (and repair):
123082023-09-22T23:29:33.236Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.182336317Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1548}
123092023-09-22T23:29:33.238Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.182726872Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1548}
123102023-09-22T23:29:33.241Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.182770086Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1548}
123112023-09-22T23:29:33.244Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.201723417Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1548}
123122023-09-22T23:29:33.247Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.201775599Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1548}
123132023-09-22T23:29:33.249Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.201814102Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1548}
123142023-09-22T23:29:33.252Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.212957388Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1548}
123152023-09-22T23:29:33.255Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.213008902Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1548}
123162023-09-22T23:29:33.258Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.213046693Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1548}
123172023-09-22T23:29:33.261ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
123182023-09-22T23:29:33.264Z 0 000-019 80 80 80 123 123 123 F F F
123192023-09-22T23:29:33.267Z 1 020-039 93 93 93 139 139 139 F F F
123202023-09-22T23:29:33.270Z 2 040-059 93 93 93 139 139 139 F F F
123212023-09-22T23:29:33.273Z 3 060-079 90 90 90 137 137 137 F F F
123222023-09-22T23:29:33.276Z 4 080-099 90 90 90 137 137 137 F F F
123232023-09-22T23:29:33.278Z 5 100-119 93 93 93 139 139 139 F F F
123242023-09-22T23:29:33.282Z 6 120-139 85 85 85 128 128 128 F F F
123252023-09-22T23:29:33.284Z 7 140-159 88 88 88 132 132 132 F F F
123262023-09-22T23:29:33.287Z 8 160-179 86 86 86 130 130 130 F F F
123272023-09-22T23:29:33.290Z 9 180-199 92 92 92 138 138 138 F F F
123282023-09-22T23:29:33.293Z 10 200-219 93 93 93 139 139 139 F F F
123292023-09-22T23:29:33.296Z 11 220-239 93 93 93 139 139 139 F F F
123302023-09-22T23:29:33.299Z 12 240-259 93 93 93 139 139 139 F F F
123312023-09-22T23:29:33.302Z 13 260-279 93 93 93 139 139 139 F F F
123322023-09-22T23:29:33.305Z 14 280-299 90 90 90 137 137 137 F F F
123332023-09-22T23:29:33.308Z 15 300-319 83 83 83 126 126 126 F F F
123342023-09-22T23:29:33.310Z 16 320-339 92 92 92 138 138 138 F F F
123352023-09-22T23:29:33.318Z 17 340-359 92 92 92 138 138 138 F F F
123362023-09-22T23:29:33.321Z 18 360-379 90 90 90 137 137 137 F F F
123372023-09-22T23:29:33.324Z 19 380-399 93 93 93 139 139 139 F F F
123382023-09-22T23:29:33.327Z 20 400-419 93 93 93 139 139 139 F F F
123392023-09-22T23:29:33.329Z 21 420-439 92 92 92 138 138 138 F F F
123402023-09-22T23:29:33.333Z 22 440-459 90 90 90 137 137 137 F F F
123412023-09-22T23:29:33.335Z 23 460-479 90 90 90 137 137 137 F F F
123422023-09-22T23:29:33.338Z 24 480-499 90 90 90 137 137 137 F F F
123432023-09-22T23:29:33.341Z 25 500-519 93 93 93 139 139 139 F F F
123442023-09-22T23:29:33.344Z 26 520-539 84 84 84 127 127 127 F F F
123452023-09-22T23:29:33.346Z 27 540-559 85 85 85 128 128 128 F F F
123462023-09-22T23:29:33.349Z 28 560-579 86 86 86 130 130 130 F F F
123472023-09-22T23:29:33.352Z 29 580-599 92 92 92 138 138 138 F F F
123482023-09-22T23:29:33.355ZMax gen: 93, Max flush: 139
123492023-09-22T23:29:33.358Z
123502023-09-22T23:29:33.361ZBegin loop 47
123512023-09-22T23:29:33.364ZDownstairs to restart: 2
123522023-09-22T23:29:33.367Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.262553038Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123532023-09-22T23:29:33.370Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.262916066Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123542023-09-22T23:29:33.373Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.262981666Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123552023-09-22T23:29:33.376Z{"msg":"Crucible stats registered with UUID: b1c4d4df-177d-4655-a7e2-bf3fe25a083b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.26300565Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123562023-09-22T23:29:33.379Z{"msg":"Crucible b1c4d4df-177d-4655-a7e2-bf3fe25a083b has session id: 3e79bdaa-f4cd-41f1-b7ec-940302704a03","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.263026637Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123572023-09-22T23:29:33.382ZCrucible runtime is spawned
123582023-09-22T23:29:33.385ZThe guest has requested activation with gen:94
123592023-09-22T23:29:33.388Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.263408053Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550,"task":"up_listen"}
123602023-09-22T23:29:33.391Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.26347108Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123612023-09-22T23:29:33.394Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.263513394Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123622023-09-22T23:29:33.397Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.26355277Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550,"looper":"2"}
123632023-09-22T23:29:33.400Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.263682064Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550,"looper":"0"}
123642023-09-22T23:29:33.403Z{"msg":"[0] b1c4d4df-177d-4655-a7e2-bf3fe25a083b looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.263954681Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550,"looper":"0"}
123652023-09-22T23:29:33.406Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.264022458Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550,"looper":"1"}
123662023-09-22T23:29:33.409Z{"msg":"[1] b1c4d4df-177d-4655-a7e2-bf3fe25a083b looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.264208644Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550,"looper":"1"}
123672023-09-22T23:29:33.412Z{"msg":"b1c4d4df-177d-4655-a7e2-bf3fe25a083b active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.264494754Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123682023-09-22T23:29:33.414Z{"msg":"Set desired generation to :94","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.264550784Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123692023-09-22T23:29:33.417Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.264685614Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123702023-09-22T23:29:33.420Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.265063718Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123712023-09-22T23:29:33.423Z{"msg":"[0] b1c4d4df-177d-4655-a7e2-bf3fe25a083b (3ad79ee5-2721-4fd8-a973-60aaab78b898) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.265354476Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123722023-09-22T23:29:33.426Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.265408656Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123732023-09-22T23:29:33.429Z{"msg":"[0] client is_active_req TRUE, promote! session 3ad79ee5-2721-4fd8-a973-60aaab78b898","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.265452657Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123742023-09-22T23:29:33.435Z{"msg":"[1] b1c4d4df-177d-4655-a7e2-bf3fe25a083b (3ad79ee5-2721-4fd8-a973-60aaab78b898) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.265595193Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123752023-09-22T23:29:33.439Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.265664941Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123762023-09-22T23:29:33.452Z{"msg":"[1] client is_active_req TRUE, promote! session 3ad79ee5-2721-4fd8-a973-60aaab78b898","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.265688099Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123772023-09-22T23:29:33.456Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.266103538Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123782023-09-22T23:29:33.459Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.26615817Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123792023-09-22T23:29:33.461Z{"msg":"b1c4d4df-177d-4655-a7e2-bf3fe25a083b WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.266299875Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123802023-09-22T23:29:33.471Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.266437446Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123812023-09-22T23:29:33.475Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.266502521Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123822023-09-22T23:29:33.478Z{"msg":"b1c4d4df-177d-4655-a7e2-bf3fe25a083b WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.2665446Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123832023-09-22T23:29:33.482Z{"msg":"[0] b1c4d4df-177d-4655-a7e2-bf3fe25a083b (3ad79ee5-2721-4fd8-a973-60aaab78b898) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.268845559Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123842023-09-22T23:29:33.485Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.268925674Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123852023-09-22T23:29:33.488Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:29:33.268996109Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123862023-09-22T23:29:33.491Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.269036703Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123872023-09-22T23:29:33.494Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.269130757Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123882023-09-22T23:29:33.497Z{"msg":"b1c4d4df-177d-4655-a7e2-bf3fe25a083b WaitQuorum WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.269171554Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123892023-09-22T23:29:33.500Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.269207034Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123902023-09-22T23:29:33.503Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.269238406Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123912023-09-22T23:29:33.506Z{"msg":"[1] b1c4d4df-177d-4655-a7e2-bf3fe25a083b (3ad79ee5-2721-4fd8-a973-60aaab78b898) WaitQuorum WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.269278433Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123922023-09-22T23:29:33.509Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.269317587Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123932023-09-22T23:29:33.513Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:29:33.269354351Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123942023-09-22T23:29:33.516Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.269390204Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123952023-09-22T23:29:33.519Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.269417407Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123962023-09-22T23:29:33.522Z{"msg":"b1c4d4df-177d-4655-a7e2-bf3fe25a083b WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.269436285Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123972023-09-22T23:29:33.525Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.269453756Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123982023-09-22T23:29:33.528Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:33.269470819Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
123992023-09-22T23:29:34.269Z{"msg":"[2] b1c4d4df-177d-4655-a7e2-bf3fe25a083b looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.265517247Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550,"looper":"2"}
124002023-09-22T23:29:34.279Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.265634422Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124012023-09-22T23:29:34.286Z{"msg":"[2] b1c4d4df-177d-4655-a7e2-bf3fe25a083b (3ad79ee5-2721-4fd8-a973-60aaab78b898) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.266780075Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124022023-09-22T23:29:34.290Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.266842658Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124032023-09-22T23:29:34.295Z{"msg":"[2] client is_active_req TRUE, promote! session 3ad79ee5-2721-4fd8-a973-60aaab78b898","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.266888321Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124042023-09-22T23:29:34.299Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.267455101Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124052023-09-22T23:29:34.303Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.267512799Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124062023-09-22T23:29:34.306Z{"msg":"b1c4d4df-177d-4655-a7e2-bf3fe25a083b WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.267556403Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124072023-09-22T23:29:34.309Z{"msg":"[2] b1c4d4df-177d-4655-a7e2-bf3fe25a083b (3ad79ee5-2721-4fd8-a973-60aaab78b898) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.269836024Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124082023-09-22T23:29:34.312Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.26988782Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124092023-09-22T23:29:34.315Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:29:34.269935469Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124102023-09-22T23:29:34.318Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.269956224Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124112023-09-22T23:29:34.321Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.269976608Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124122023-09-22T23:29:34.323Z{"msg":"b1c4d4df-177d-4655-a7e2-bf3fe25a083b WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.270019157Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124132023-09-22T23:29:34.326Z{"msg":"[0]R flush_numbers[0..12]: [123, 139, 139, 137, 137, 139, 128, 132, 130, 138, 139, 139]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.270060135Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124142023-09-22T23:29:34.329Z{"msg":"[0]R generation[0..12]: [80, 93, 93, 90, 90, 93, 85, 88, 86, 92, 93, 93]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.270100017Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124152023-09-22T23:29:34.332Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.270140715Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124162023-09-22T23:29:34.335Z{"msg":"[1]R flush_numbers[0..12]: [123, 139, 139, 137, 137, 139, 128, 132, 130, 138, 139, 139]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.270180549Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124172023-09-22T23:29:34.338Z{"msg":"[1]R generation[0..12]: [80, 93, 93, 90, 90, 93, 85, 88, 86, 92, 93, 93]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.270225255Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124182023-09-22T23:29:34.341Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.270266102Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124192023-09-22T23:29:34.344Z{"msg":"[2]R flush_numbers[0..12]: [123, 139, 139, 137, 137, 139, 128, 132, 130, 138, 139, 139]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:94
124202023-09-22T23:29:34.347ZWait for a query_work_queue command to finish before sending IO
124212023-09-22T23:29:34.350Z,"time":"2023-09-22T23:29:34.270372536Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124222023-09-22T23:29:34.353Z{"msg":"[2]R generation[0..12]: [80, 93, 93, 90, 90, 93, 85, 88, 86, 92, 93, 93]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.270563669Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124232023-09-22T23:29:34.356Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.270630826Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124242023-09-22T23:29:34.359Z{"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.270675414Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124252023-09-22T23:29:34.362Z{"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.270714893Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124262023-09-22T23:29:34.364Z{"msg":"Next flush: 140","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.270753329Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124272023-09-22T23:29:34.367Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.270807782Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124282023-09-22T23:29:34.370Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.270856135Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124292023-09-22T23:29:34.373Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.270896292Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124302023-09-22T23:29:34.376Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.270937825Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124312023-09-22T23:29:34.379Z{"msg":"b1c4d4df-177d-4655-a7e2-bf3fe25a083b is now active with session: 3ad79ee5-2721-4fd8-a973-60aaab78b898","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.27099682Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124322023-09-22T23:29:34.384Z{"msg":"b1c4d4df-177d-4655-a7e2-bf3fe25a083b Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.271055733Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124332023-09-22T23:29:34.393Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.271097616Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124342023-09-22T23:29:34.396Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.271140334Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124352023-09-22T23:29:34.399Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.271196749Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124362023-09-22T23:29:34.402Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.271272143Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124372023-09-22T23:29:34.404Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.271315691Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124382023-09-22T23:29:34.408Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.271356365Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124392023-09-22T23:29:34.410Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.27137622Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124402023-09-22T23:29:34.413Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.271439156Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124412023-09-22T23:29:34.416Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
124422023-09-22T23:29:34.419Z,"time":"2023-09-22T23:29:34.271500704Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124432023-09-22T23:29:34.422Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.271605346Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124442023-09-22T23:29:34.425Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.271650118Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1550}
124452023-09-22T23:29:34.640ZLoading write count information from file "/tmp/test_repair_verify.data"
124462023-09-22T23:29:34.643ZRun Repair workload
124472023-09-22T23:29:34.646Z01/30 Write block 411 len 36864 data: 7 4 6 8 8 9 9 8 9
124482023-09-22T23:29:34.649Z02/30 Write block 568 len 12288 data: 13 13 9
124492023-09-22T23:29:34.651Z03/30 Write block 486 len 24576 data: 11 9 13 14 13 14
124502023-09-22T23:29:34.654Z04/30 Flush
124512023-09-22T23:29:34.658Z05/30 Flush
124522023-09-22T23:29:34.660Z06/30 Write block 293 len 16384 data: 10 8 9 8
124532023-09-22T23:29:34.663Z07/30 Write block 70 len 24576 data: 8 7 7 4 5 7
124542023-09-22T23:29:34.666Z08/30 Flush
124552023-09-22T23:29:34.669Z09/30 Write block 468 len 36864 data: 10 9 7 5 5 5 5 5 7
124562023-09-22T23:29:34.672Z10/30 Read block 392 len 40960
124572023-09-22T23:29:34.674Z11/30 Write block 312 len 12288 data: 3 5 7
124582023-09-22T23:29:34.677Z12/30 Write block 271 len 36864 data: 13 13 11 11 13 16 12 15 15
124592023-09-22T23:29:34.680Z13/30 Write block 155 len 20480 data: 6 5 6 9 10
124602023-09-22T23:29:34.683Z14/30 Write block 589 len 40960 data: 7 8 9 11 12 9 9 9 5 4
124612023-09-22T23:29:34.686Z15/30 Write block 441 len 4096 data: 7
124622023-09-22T23:29:34.689Z16/30 Write block 164 len 20480 data: 8 7 5 5 4
124632023-09-22T23:29:34.693Z17/30 Write block 106 len 36864 data: 13 12 12 12 12 9 8 8 6
124642023-09-22T23:29:34.701Z18/30 Read block 313 len 4096
124652023-09-22T23:29:34.704Z19/30 Read block 22 len 36864
124662023-09-22T23:29:34.709Z20/30 Read block 120 len 36864
124672023-09-22T23:29:34.714Z21/30 Write block 442 len 16384 data: 8 10 10 10
124682023-09-22T23:29:34.718Z22/30 Read block 527 len 40960
124692023-09-22T23:29:34.724Z23/30 Write block 423 len 16384 data: 5 5 3 5
124702023-09-22T23:29:34.728Z24/30 Write block 464 len 24576 data: 9 13 10 10 11 10
124712023-09-22T23:29:34.732Z25/30 Flush
124722023-09-22T23:29:34.735Z26/30 Write block 195 len 12288 data: 10 10 9
124732023-09-22T23:29:34.740Z27/30 Read block 282 len 40960
124742023-09-22T23:29:34.746Z28/30 Write block 512 len 40960 data: 9 11 13 12 12 12 12 10 9 7
124752023-09-22T23:29:34.752Z29/30 Read block 190 len 12288
124762023-09-22T23:29:34.757Z30/30 Write block 219 len 28672 data: 5 7 6 6 7 6 7
124772023-09-22T23:29:34.763Z----------------------------------------------------------------
124782023-09-22T23:29:34.768Z Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:25
124792023-09-22T23:29:34.775ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
124802023-09-22T23:29:34.779Z 6 Acked 1005 Write 4 Done Done Sent false
124812023-09-22T23:29:34.789Z 7 Acked 1006 Write 6 Done Done Sent false
124822023-09-22T23:29:34.792Z 8 Acked 1007 Flush 0 Done Done Sent false
124832023-09-22T23:29:34.794Z 9 Acked 1008 Write 9 Done Done Sent false
124842023-09-22T23:29:34.797Z 10 Acked 1009 Read 10 Done Done Sent false
124852023-09-22T23:29:34.800Z 11 Acked 1010 Write 3 Done Done Sent false
124862023-09-22T23:29:34.803Z 12 Acked 1011 Write 9 Done Done Sent false
124872023-09-22T23:29:34.805Z 13 Acked 1012 Write 5 Done Done Sent false
124882023-09-22T23:29:34.808Z 14 Acked 1013 Write 10 Done Done Sent false
124892023-09-22T23:29:34.811Z 15 Acked 1014 Write 1 Done Done Sent false
124902023-09-22T23:29:34.814Z 16 Acked 1015 Write 5 Done Done Sent false
124912023-09-22T23:29:34.818Z 17 Acked 1016 Write 9 Done Done Sent false
124922023-09-22T23:29:34.821Z 18 Acked 1017 Read 1 Done Done Sent false
124932023-09-22T23:29:34.824Z 19 Acked 1018 Read 9 Done Done Sent false
124942023-09-22T23:29:34.827Z 20 Acked 1019 Read 9 Done Done Sent false
124952023-09-22T23:29:34.830Z 21 Acked 1020 Write 4 Done Done Sent false
124962023-09-22T23:29:34.832Z 22 Acked 1021 Read 10 Done Done Sent false
124972023-09-22T23:29:34.836Z 23 Acked 1022 Write 4 Done Done Sent false
124982023-09-22T23:29:34.839Z 24 Acked 1023 Write 6 Done Done Sent false
124992023-09-22T23:29:34.841Z 25 Acked 1024 Flush 0 Done Done Sent false
125002023-09-22T23:29:34.844Z 26 Acked 1025 Write 3 Done Done Sent false
125012023-09-22T23:29:34.868Z 27 Acked 1026 Read 10 Done Done Sent false
125022023-09-22T23:29:34.870Z 28 Acked 1027 Write 10 Done Done Sent false
125032023-09-22T23:29:34.873Z 29 Acked 1028 Read 3 Done Sent Sent false
125042023-09-22T23:29:34.876Z 30 Acked 1029 Write 7 Sent Sent Sent false
125052023-09-22T23:29:34.878Z STATES DS:0 DS:1 DS:2 TOTAL
125062023-09-22T23:29:34.881Z New 0 0 0 0
125072023-09-22T23:29:34.884Z Sent 1 2 25 28
125082023-09-22T23:29:34.886Z Done 24 23 0 47
125092023-09-22T23:29:34.889Z Skipped 0 0 0 0
125102023-09-22T23:29:34.891Z Error 0 0 0 0
125112023-09-22T23:29:34.894ZError: Difference in extent metadata found!
125122023-09-22T23:29:34.897ZLast Flush: 1024 1024 1004
125132023-09-22T23:29:34.900ZDownstairs last five completed: 1004 1003 1002 1001 1000
125142023-09-22T23:29:34.902ZUpstairs last five completed: 30 29 28 27 26 25
125152023-09-22T23:29:34.905ZWrote out file "/tmp/test_repair_verify.data"
125162023-09-22T23:29:34.908Z
125172023-09-22T23:29:34.910ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
125182023-09-22T23:29:34.913Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.410551847Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1551}
125192023-09-22T23:29:34.916Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.410878194Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1551}
125202023-09-22T23:29:34.919Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.410901716Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1551}
125212023-09-22T23:29:34.922Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.43751801Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1551}
125222023-09-22T23:29:34.925Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.43755025Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1551}
125232023-09-22T23:29:34.928Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.437569312Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1551}
125242023-09-22T23:29:34.931Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.454811535Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1551}
125252023-09-22T23:29:34.934Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.454839394Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1551}
125262023-09-22T23:29:34.937Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:34.454857536Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1551}
125272023-09-22T23:29:34.939ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
125282023-09-22T23:29:34.942Z 0 000-019 80 80 80 123 123 123 F F F
125292023-09-22T23:29:34.945Z 1 020-039 93 93 93 139 139 139 F F F
125302023-09-22T23:29:34.948Z 2 040-059 93 93 93 139 139 139 F F F
125312023-09-22T23:29:34.951Z 3 060-079 94 94 90 142 142 137 F F F <---
125322023-09-22T23:29:34.954Z 4 080-099 90 90 90 137 137 137 F F F
125332023-09-22T23:29:34.956Z 5 100-119 94 94 93 143 143 139 F F F <---
125342023-09-22T23:29:34.959Z 6 120-139 85 85 85 128 128 128 F F F
125352023-09-22T23:29:34.962Z 7 140-159 94 94 88 143 143 132 F F F <---
125362023-09-22T23:29:34.965Z 8 160-179 94 94 86 143 143 130 F F F <---
125372023-09-22T23:29:34.968Z 9 180-199 92 92 92 138 138 138 T T F <---
125382023-09-22T23:29:34.972Z 10 200-219 93 93 93 139 139 139 T T F <---
125392023-09-22T23:29:34.974Z 11 220-239 93 93 93 139 139 139 T T F <---
125402023-09-22T23:29:34.977Z 12 240-259 93 93 93 139 139 139 F F F
125412023-09-22T23:29:34.980Z 13 260-279 94 94 93 143 143 139 F F F <---
125422023-09-22T23:29:34.983Z 14 280-299 94 94 90 142 142 137 F F F <---
125432023-09-22T23:29:34.986Z 15 300-319 94 94 83 143 143 126 F F F <---
125442023-09-22T23:29:34.988Z 16 320-339 92 92 92 138 138 138 F F F
125452023-09-22T23:29:34.991Z 17 340-359 92 92 92 138 138 138 F F F
125462023-09-22T23:29:34.994Z 18 360-379 90 90 90 137 137 137 F F F
125472023-09-22T23:29:34.997Z 19 380-399 93 93 93 139 139 139 F F F
125482023-09-22T23:29:35.000Z 20 400-419 94 94 94 140 140 140 F F F
125492023-09-22T23:29:35.003Z 21 420-439 94 94 92 143 143 138 F F F <---
125502023-09-22T23:29:35.006Z 22 440-459 94 94 90 143 143 137 F F F <---
125512023-09-22T23:29:35.009Z 23 460-479 94 94 90 143 143 137 F F F <---
125522023-09-22T23:29:35.011Z 24 480-499 94 94 94 140 140 140 F F F
125532023-09-22T23:29:35.014Z 25 500-519 93 93 93 139 139 139 T T F <---
125542023-09-22T23:29:35.017Z 26 520-539 84 84 84 127 127 127 T T F <---
125552023-09-22T23:29:35.019Z 27 540-559 85 85 85 128 128 128 F F F
125562023-09-22T23:29:35.022Z 28 560-579 94 94 94 140 140 140 F F F
125572023-09-22T23:29:35.025Z 29 580-599 94 94 92 143 143 138 F F F <---
125582023-09-22T23:29:35.028ZMax gen: 94, Max flush: 143
125592023-09-22T23:29:35.031ZA Difference in extent metadata is expected here
125602023-09-22T23:29:35.033ZOn loop 47
125612023-09-22T23:29:35.036Z
125622023-09-22T23:29:35.039ZVerifying data now
125632023-09-22T23:29:41.158ZNo change after verify
125642023-09-22T23:29:41.162ZLoop: 47 Downstairs dump after verify (and repair):
125652023-09-22T23:29:41.176Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.176037047Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1556}
125662023-09-22T23:29:41.179Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.176469327Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1556}
125672023-09-22T23:29:41.182Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.176494554Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1556}
125682023-09-22T23:29:41.196Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.195305567Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1556}
125692023-09-22T23:29:41.199Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.195358039Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1556}
125702023-09-22T23:29:41.202Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.195378537Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1556}
125712023-09-22T23:29:41.206Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.206410687Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1556}
125722023-09-22T23:29:41.209Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.206461049Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1556}
125732023-09-22T23:29:41.212Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.206481206Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1556}
125742023-09-22T23:29:41.218ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
125752023-09-22T23:29:41.221Z 0 000-019 80 80 80 123 123 123 F F F
125762023-09-22T23:29:41.224Z 1 020-039 93 93 93 139 139 139 F F F
125772023-09-22T23:29:41.227Z 2 040-059 93 93 93 139 139 139 F F F
125782023-09-22T23:29:41.230Z 3 060-079 94 94 94 142 142 142 F F F
125792023-09-22T23:29:41.233Z 4 080-099 90 90 90 137 137 137 F F F
125802023-09-22T23:29:41.236Z 5 100-119 94 94 94 143 143 143 F F F
125812023-09-22T23:29:41.239Z 6 120-139 85 85 85 128 128 128 F F F
125822023-09-22T23:29:41.242Z 7 140-159 94 94 94 143 143 143 F F F
125832023-09-22T23:29:41.245Z 8 160-179 94 94 94 143 143 143 F F F
125842023-09-22T23:29:41.248Z 9 180-199 95 95 95 144 144 144 F F F
125852023-09-22T23:29:41.251Z 10 200-219 95 95 95 144 144 144 F F F
125862023-09-22T23:29:41.254Z 11 220-239 95 95 95 144 144 144 F F F
125872023-09-22T23:29:41.257Z 12 240-259 93 93 93 139 139 139 F F F
125882023-09-22T23:29:41.260Z 13 260-279 94 94 94 143 143 143 F F F
125892023-09-22T23:29:41.263Z 14 280-299 94 94 94 142 142 142 F F F
125902023-09-22T23:29:41.267Z 15 300-319 94 94 94 143 143 143 F F F
125912023-09-22T23:29:41.271Z 16 320-339 92 92 92 138 138 138 F F F
125922023-09-22T23:29:41.277Z 17 340-359 92 92 92 138 138 138 F F F
125932023-09-22T23:29:41.280Z 18 360-379 90 90 90 137 137 137 F F F
125942023-09-22T23:29:41.283Z 19 380-399 93 93 93 139 139 139 F F F
125952023-09-22T23:29:41.286Z 20 400-419 94 94 94 140 140 140 F F F
125962023-09-22T23:29:41.290Z 21 420-439 94 94 94 143 143 143 F F F
125972023-09-22T23:29:41.293Z 22 440-459 94 94 94 143 143 143 F F F
125982023-09-22T23:29:41.296Z 23 460-479 94 94 94 143 143 143 F F F
125992023-09-22T23:29:41.299Z 24 480-499 94 94 94 140 140 140 F F F
126002023-09-22T23:29:41.302Z 25 500-519 95 95 95 144 144 144 F F F
126012023-09-22T23:29:41.305Z 26 520-539 95 95 95 144 144 144 F F F
126022023-09-22T23:29:41.308Z 27 540-559 85 85 85 128 128 128 F F F
126032023-09-22T23:29:41.311Z 28 560-579 94 94 94 140 140 140 F F F
126042023-09-22T23:29:41.314Z 29 580-599 94 94 94 143 143 143 F F F
126052023-09-22T23:29:41.317ZMax gen: 95, Max flush: 144
126062023-09-22T23:29:41.320Z
126072023-09-22T23:29:41.323ZBegin loop 48
126082023-09-22T23:29:41.326ZDownstairs to restart: 1
126092023-09-22T23:29:41.330Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.25450859Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126102023-09-22T23:29:41.332Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.254895594Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126112023-09-22T23:29:41.336Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.254949265Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126122023-09-22T23:29:41.339Z{"msg":"Crucible stats registered with UUID: 87157f54-fc80-4877-b398-cdf4ce6d5b77","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.254973142Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126132023-09-22T23:29:41.345Z{"msg":"Crucible 87157f54-fc80-4877-b398-cdf4ce6d5b77 has session id: fbe1df32-33cf-437c-8ca5-e91930a77164","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.255014322Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126142023-09-22T23:29:41.349ZCrucible runtime is spawned
126152023-09-22T23:29:41.359ZThe guest has requested activation with gen:96
126162023-09-22T23:29:41.372Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.255264875Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558,"task":"up_listen"}
126172023-09-22T23:29:41.380Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.255333121Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126182023-09-22T23:29:41.383Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.255375685Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126192023-09-22T23:29:41.386Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.255414562Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558,"looper":"1"}
126202023-09-22T23:29:41.389Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.255566062Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558,"looper":"2"}
126212023-09-22T23:29:41.392Z{"msg":"[2] 87157f54-fc80-4877-b398-cdf4ce6d5b77 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.255761652Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558,"looper":"2"}
126222023-09-22T23:29:41.395Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.255857263Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558,"looper":"0"}
126232023-09-22T23:29:41.398Z{"msg":"[0] 87157f54-fc80-4877-b398-cdf4ce6d5b77 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.25602789Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558,"looper":"0"}
126242023-09-22T23:29:41.401Z{"msg":"87157f54-fc80-4877-b398-cdf4ce6d5b77 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.256403793Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126252023-09-22T23:29:41.404Z{"msg":"Set desired generation to :96","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.256456003Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126262023-09-22T23:29:41.407Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.256614906Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126272023-09-22T23:29:41.410Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.256951701Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126282023-09-22T23:29:41.413Z{"msg":"[2] 87157f54-fc80-4877-b398-cdf4ce6d5b77 (dd4b4dcb-3f91-48f0-aa93-73551e72f1ab) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.257339175Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126292023-09-22T23:29:41.416Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.257381152Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126302023-09-22T23:29:41.418Z{"msg":"[2] client is_active_req TRUE, promote! session dd4b4dcb-3f91-48f0-aa93-73551e72f1ab","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.257403089Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126312023-09-22T23:29:41.421Z{"msg":"[0] 87157f54-fc80-4877-b398-cdf4ce6d5b77 (dd4b4dcb-3f91-48f0-aa93-73551e72f1ab) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.257523862Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126322023-09-22T23:29:41.424Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.257679844Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126332023-09-22T23:29:41.427Z{"msg":"[0] client is_active_req TRUE, promote! session dd4b4dcb-3f91-48f0-aa93-73551e72f1ab","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.257733067Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126342023-09-22T23:29:41.430Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.258014937Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126352023-09-22T23:29:41.433Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.258079162Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126362023-09-22T23:29:41.436Z{"msg":"87157f54-fc80-4877-b398-cdf4ce6d5b77 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.258121534Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126372023-09-22T23:29:41.439Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.25821844Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126382023-09-22T23:29:41.442Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.258289077Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126392023-09-22T23:29:41.445Z{"msg":"87157f54-fc80-4877-b398-cdf4ce6d5b77 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.258360274Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126402023-09-22T23:29:41.447Z{"msg":"[2] 87157f54-fc80-4877-b398-cdf4ce6d5b77 (dd4b4dcb-3f91-48f0-aa93-73551e72f1ab) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.26046083Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126412023-09-22T23:29:41.451Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.260513719Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126422023-09-22T23:29:41.454Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:29:41.260585345Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126432023-09-22T23:29:41.457Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.260658688Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126442023-09-22T23:29:41.460Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.260728874Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126452023-09-22T23:29:41.463Z{"msg":"87157f54-fc80-4877-b398-cdf4ce6d5b77 WaitActive New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.260809631Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126462023-09-22T23:29:41.466Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.260885372Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126472023-09-22T23:29:41.469Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.260978089Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126482023-09-22T23:29:41.472Z{"msg":"[0] 87157f54-fc80-4877-b398-cdf4ce6d5b77 (dd4b4dcb-3f91-48f0-aa93-73551e72f1ab) WaitActive New WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.261055949Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126492023-09-22T23:29:41.475Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.261104213Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126502023-09-22T23:29:41.478Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:29:41.26114395Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126512023-09-22T23:29:41.481Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.261182315Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126522023-09-22T23:29:41.484Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.261232982Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126532023-09-22T23:29:41.487Z{"msg":"87157f54-fc80-4877-b398-cdf4ce6d5b77 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.261276085Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126542023-09-22T23:29:41.490Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.261314793Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126552023-09-22T23:29:41.493Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:41.261334139Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126562023-09-22T23:29:42.257Z{"msg":"[1] 87157f54-fc80-4877-b398-cdf4ce6d5b77 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.257007303Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558,"looper":"1"}
126572023-09-22T23:29:42.260Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.257123039Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126582023-09-22T23:29:42.263Z{"msg":"[1] 87157f54-fc80-4877-b398-cdf4ce6d5b77 (dd4b4dcb-3f91-48f0-aa93-73551e72f1ab) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.258276905Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126592023-09-22T23:29:42.268Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.258331026Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126602023-09-22T23:29:42.274Z{"msg":"[1] client is_active_req TRUE, promote! session dd4b4dcb-3f91-48f0-aa93-73551e72f1ab","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.258395142Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126612023-09-22T23:29:42.277Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.259064861Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126622023-09-22T23:29:42.280Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.259145533Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126632023-09-22T23:29:42.283Z{"msg":"87157f54-fc80-4877-b398-cdf4ce6d5b77 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.259189878Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126642023-09-22T23:29:42.286Z{"msg":"[1] 87157f54-fc80-4877-b398-cdf4ce6d5b77 (dd4b4dcb-3f91-48f0-aa93-73551e72f1ab) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.261502409Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126652023-09-22T23:29:42.289Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.261662625Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126662023-09-22T23:29:42.292Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:29:42.261731335Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126672023-09-22T23:29:42.295Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.261797563Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126682023-09-22T23:29:42.298Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.26186352Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126692023-09-22T23:29:42.301Z{"msg":"87157f54-fc80-4877-b398-cdf4ce6d5b77 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.261929255Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126702023-09-22T23:29:42.303Z{"msg":"[0]R flush_numbers[0..12]: [123, 139, 139, 142, 137, 143, 128, 143, 143, 144, 144, 144]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.262000908Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126712023-09-22T23:29:42.307Z{"msg":"The guest has finished waiting for activation with:96
126722023-09-22T23:29:42.309ZWait for a query_work_queue command to finish before sending IO
126732023-09-22T23:29:42.312Z[0]R generation[0..12]: [80, 93, 93, 94, 90, 94, 85, 94, 94, 95, 95, 95]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.262233105Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126742023-09-22T23:29:42.315Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.262314267Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126752023-09-22T23:29:42.318Z{"msg":"[1]R flush_numbers[0..12]: [123, 139, 139, 142, 137, 143, 128, 143, 143, 144, 144, 144]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.262375199Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126762023-09-22T23:29:42.323Z{"msg":"[1]R generation[0..12]: [80, 93, 93, 94, 90, 94, 85, 94, 94, 95, 95, 95]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.262420128Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126772023-09-22T23:29:42.331Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.262442565Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126782023-09-22T23:29:42.340Z{"msg":"[2]R flush_numbers[0..12]: [123, 139, 139, 142, 137, 143, 128, 143, 143, 144, 144, 144]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.26248067Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126792023-09-22T23:29:42.348Z{"msg":"[2]R generation[0..12]: [80, 93, 93, 94, 90, 94, 85, 94, 94, 95, 95, 95]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.262527957Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126802023-09-22T23:29:42.353Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.262573728Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126812023-09-22T23:29:42.357Z{"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.262619194Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126822023-09-22T23:29:42.360Z{"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.2626416Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126832023-09-22T23:29:42.363Z{"msg":"Next flush: 145","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.262689906Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126842023-09-22T23:29:42.366Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.26273847Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126852023-09-22T23:29:42.369Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.262830199Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126862023-09-22T23:29:42.372Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.262854515Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126872023-09-22T23:29:42.375Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.262903604Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126882023-09-22T23:29:42.377Z{"msg":"87157f54-fc80-4877-b398-cdf4ce6d5b77 is now active with session: dd4b4dcb-3f91-48f0-aa93-73551e72f1ab","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.262950385Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126892023-09-22T23:29:42.380Z{"msg":"87157f54-fc80-4877-b398-cdf4ce6d5b77 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.262972387Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126902023-09-22T23:29:42.383Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.263014119Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126912023-09-22T23:29:42.387Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
126922023-09-22T23:29:42.391Z2023-09-22T23:29:42.263059348Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126932023-09-22T23:29:42.395Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.26316011Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126942023-09-22T23:29:42.398Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.263210999Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126952023-09-22T23:29:42.402Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.263258394Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126962023-09-22T23:29:42.810Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.263297436Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126972023-09-22T23:29:42.813Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.263335525Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126982023-09-22T23:29:42.816Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.263373303Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
126992023-09-22T23:29:42.818Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.263414358Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
127002023-09-22T23:29:42.821Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.263508389Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
127012023-09-22T23:29:42.824Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.26354095Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1558}
127022023-09-22T23:29:42.827ZLoading write count information from file "/tmp/test_repair_verify.data"
127032023-09-22T23:29:42.829ZRun Repair workload
127042023-09-22T23:29:42.832Z01/30 Flush
127052023-09-22T23:29:42.835Z02/30 Write block 262 len 40960 data: 11 11 11 10 10 13 11 12 12 14
127062023-09-22T23:29:42.838Z03/30 Write block 484 len 12288 data: 10 10 12
127072023-09-22T23:29:42.840Z04/30 Write block 228 len 24576 data: 6 6 8 4 4 4
127082023-09-22T23:29:42.844Z05/30 Write block 271 len 16384 data: 15 14 12 12
127092023-09-22T23:29:42.846Z06/30 Write block 416 len 36864 data: 10 10 9 10 9 9 6 6 6
127102023-09-22T23:29:42.849Z07/30 Read block 432 len 8192
127112023-09-22T23:29:42.852Z08/30 Read block 258 len 40960
127122023-09-22T23:29:42.854Z09/30 Write block 65 len 8192 data: 18 17
127132023-09-22T23:29:42.857Z10/30 Write block 474 len 4096 data: 6
127142023-09-22T23:29:42.860Z11/30 Read block 86 len 12288
127152023-09-22T23:29:42.863Z12/30 Read block 32 len 16384
127162023-09-22T23:29:42.866Z13/30 Read block 77 len 8192
127172023-09-22T23:29:42.868Z14/30 Read block 244 len 20480
127182023-09-22T23:29:42.871Z15/30 Read block 533 len 24576
127192023-09-22T23:29:42.874Z16/30 Read block 52 len 32768
127202023-09-22T23:29:42.877Z17/30 Read block 239 len 12288
127212023-09-22T23:29:42.880Z18/30 Flush
127222023-09-22T23:29:42.883Z19/30 Write block 333 len 16384 data: 7 9 11 10
127232023-09-22T23:29:42.886Z20/30 Read block 29 len 28672
127242023-09-22T23:29:42.889Z21/30 Read block 303 len 24576
127252023-09-22T23:29:42.893Z22/30 Read block 264 len 32768
127262023-09-22T23:29:42.896Z23/30 Read block 263 len 12288
127272023-09-22T23:29:42.899Z24/30 Write block 205 len 20480 data: 13 14 13 15 12
127282023-09-22T23:29:42.902Z25/30 Write block 524 len 20480 data: 8 9 10 10 9
127292023-09-22T23:29:42.905Z26/30 Read block 254 len 16384
127302023-09-22T23:29:42.908Z27/30 Read block 235 len 8192
127312023-09-22T23:29:42.911Z28/30 Write block 515 len 4096 data: 13
127322023-09-22T23:29:42.914Z29/30 Read block 72 len 16384
127332023-09-22T23:29:42.918Z30/30 Read block 16 len 20480
127342023-09-22T23:29:42.921Z----------------------------------------------------------------
127352023-09-22T23:29:42.923Z Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:30
127362023-09-22T23:29:42.927ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
127372023-09-22T23:29:42.929Z 1 Acked 1000 Flush 0 Done Sent Done false
127382023-09-22T23:29:42.932Z 2 Acked 1001 Write 10 Done Sent Done false
127392023-09-22T23:29:42.935Z 3 Acked 1002 Write 3 Done Sent Done false
127402023-09-22T23:29:42.938Z 4 Acked 1003 Write 6 Done Sent Done false
127412023-09-22T23:29:42.943Z 5 Acked 1004 Write 4 Done Sent Done false
127422023-09-22T23:29:42.946Z 6 Acked 1005 Write 9 Done Sent Done false
127432023-09-22T23:29:42.949Z 7 Acked 1006 Read 2 Done Sent Done false
127442023-09-22T23:29:42.952Z 8 Acked 1007 Read 10 Done Sent Done false
127452023-09-22T23:29:42.955Z 9 Acked 1008 Write 2 Done Sent Done false
127462023-09-22T23:29:42.957Z 10 Acked 1009 Write 1 Done Sent Done false
127472023-09-22T23:29:42.960Z 11 Acked 1010 Read 3 Done Sent Done false
127482023-09-22T23:29:42.963Z 12 Acked 1011 Read 4 Done Sent Done false
127492023-09-22T23:29:42.966Z 13 Acked 1012 Read 2 Done Sent Done false
127502023-09-22T23:29:42.969Z 14 Acked 1013 Read 5 Done Sent Done false
127512023-09-22T23:29:42.972Z 15 Acked 1014 Read 6 Done Sent Done false
127522023-09-22T23:29:42.975Z 16 Acked 1015 Read 8 Done Sent Done false
127532023-09-22T23:29:42.978Z 17 Acked 1016 Read 3 Done Sent Done false
127542023-09-22T23:29:42.981Z 18 Acked 1017 Flush 0 Done Sent Done false
127552023-09-22T23:29:42.984Z 19 Acked 1018 Write 4 Done Sent Done false
127562023-09-22T23:29:42.987Z 20 Acked 1019 Read 7 Done Sent Done false
127572023-09-22T23:29:42.989Z 21 Acked 1020 Read 6 Done Sent Done false
127582023-09-22T23:29:42.992Z 22 Acked 1021 Read 8 Done Sent Done false
127592023-09-22T23:29:42.995Z 23 Acked 1022 Read 3 Done Sent Done false
127602023-09-22T23:29:43.001Z 24 Acked 1023 Write 5 Done Sent Done false
127612023-09-22T23:29:43.004Z 25 Acked 1024 Write 5 Done Sent Done false
127622023-09-22T23:29:43.007ZError: Difference in extent metadata found!
127632023-09-22T23:29:43.011Z 26 Acked 1025 Read 4 Done Sent Done false
127642023-09-22T23:29:43.014Z 27 Acked 1026 Read 2 Done Sent Done false
127652023-09-22T23:29:43.017Z 28 Acked 1027 Write 1 Done Sent Done false
127662023-09-22T23:29:43.020Z 29 Acked 1028 Read 4 Done Sent Done false
127672023-09-22T23:29:43.043Z 30 Acked 1029 Read 5 Sent Sent Done false
127682023-09-22T23:29:43.047Z STATES DS:0 DS:1 DS:2 TOTAL
127692023-09-22T23:29:43.050Z New 0 0 0 0
127702023-09-22T23:29:43.052Z Sent 1 30 0 31
127712023-09-22T23:29:43.055Z Done 29 0 30 59
127722023-09-22T23:29:43.059Z Skipped 0 0 0 0
127732023-09-22T23:29:43.063Z Error 0 0 0 0
127742023-09-22T23:29:43.066ZLast Flush: 1017 0 1017
127752023-09-22T23:29:43.069ZDownstairs last five completed:
127762023-09-22T23:29:43.072ZUpstairs last five completed: 30 29 28 27 26 25
127772023-09-22T23:29:43.075ZWrote out file "/tmp/test_repair_verify.data"
127782023-09-22T23:29:43.078Z
127792023-09-22T23:29:43.081ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
127802023-09-22T23:29:43.084Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.387607591Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1559}
127812023-09-22T23:29:43.087Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.38802382Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1559}
127822023-09-22T23:29:43.090Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.38804735Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1559}
127832023-09-22T23:29:43.093Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.413690795Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1559}
127842023-09-22T23:29:43.096Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.413743454Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1559}
127852023-09-22T23:29:43.099Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.413762582Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1559}
127862023-09-22T23:29:43.101Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.435221154Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1559}
127872023-09-22T23:29:43.104Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.435259045Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1559}
127882023-09-22T23:29:43.107Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:42.435277613Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1559}
127892023-09-22T23:29:43.110ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
127902023-09-22T23:29:43.113Z 0 000-019 80 80 80 123 123 123 F F F
127912023-09-22T23:29:43.116Z 1 020-039 93 93 93 139 139 139 F F F
127922023-09-22T23:29:43.119Z 2 040-059 93 93 93 139 139 139 F F F
127932023-09-22T23:29:43.122Z 3 060-079 96 94 96 146 142 146 F F F <---
127942023-09-22T23:29:43.125Z 4 080-099 90 90 90 137 137 137 F F F
127952023-09-22T23:29:43.128Z 5 100-119 94 94 94 143 143 143 F F F
127962023-09-22T23:29:43.131Z 6 120-139 85 85 85 128 128 128 F F F
127972023-09-22T23:29:43.134Z 7 140-159 94 94 94 143 143 143 F F F
127982023-09-22T23:29:43.137Z 8 160-179 94 94 94 143 143 143 F F F
127992023-09-22T23:29:43.140Z 9 180-199 95 95 95 144 144 144 F F F
128002023-09-22T23:29:43.147Z 10 200-219 95 95 95 144 144 144 T F T <---
128012023-09-22T23:29:43.152Z 11 220-239 96 95 96 146 144 146 F F F <---
128022023-09-22T23:29:43.157Z 12 240-259 93 93 93 139 139 139 F F F
128032023-09-22T23:29:43.161Z 13 260-279 96 94 96 146 143 146 F F F <---
128042023-09-22T23:29:43.165Z 14 280-299 94 94 94 142 142 142 F F F
128052023-09-22T23:29:43.168Z 15 300-319 94 94 94 143 143 143 F F F
128062023-09-22T23:29:43.171Z 16 320-339 92 92 92 138 138 138 T F T <---
128072023-09-22T23:29:43.174Z 17 340-359 92 92 92 138 138 138 F F F
128082023-09-22T23:29:43.178Z 18 360-379 90 90 90 137 137 137 F F F
128092023-09-22T23:29:43.181Z 19 380-399 93 93 93 139 139 139 F F F
128102023-09-22T23:29:43.185Z 20 400-419 96 94 96 146 140 146 F F F <---
128112023-09-22T23:29:43.188Z 21 420-439 96 94 96 146 143 146 F F F <---
128122023-09-22T23:29:43.191Z 22 440-459 94 94 94 143 143 143 F F F
128132023-09-22T23:29:43.195Z 23 460-479 96 94 96 146 143 146 F F F <---
128142023-09-22T23:29:43.198Z 24 480-499 96 94 96 146 140 146 F F F <---
128152023-09-22T23:29:43.202Z 25 500-519 95 95 95 144 144 144 T F T <---
128162023-09-22T23:29:43.205Z 26 520-539 95 95 95 144 144 144 T F T <---
128172023-09-22T23:29:43.208Z 27 540-559 85 85 85 128 128 128 F F F
128182023-09-22T23:29:43.212Z 28 560-579 94 94 94 140 140 140 F F F
128192023-09-22T23:29:43.215Z 29 580-599 94 94 94 143 143 143 F F F
128202023-09-22T23:29:43.218ZMax gen: 96, Max flush: 146
128212023-09-22T23:29:43.222ZA Difference in extent metadata is expected here
128222023-09-22T23:29:43.225ZOn loop 48
128232023-09-22T23:29:43.228Z
128242023-09-22T23:29:43.231ZVerifying data now
128252023-09-22T23:29:48.693ZNo change after verify
128262023-09-22T23:29:48.696ZLoop: 48 Downstairs dump after verify (and repair):
128272023-09-22T23:29:48.713Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.712353969Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1564}
128282023-09-22T23:29:48.716Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.712666885Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1564}
128292023-09-22T23:29:48.719Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.712691478Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1564}
128302023-09-22T23:29:48.731Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.731547403Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1564}
128312023-09-22T23:29:48.734Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.731599935Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1564}
128322023-09-22T23:29:48.737Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.731620623Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1564}
128332023-09-22T23:29:48.743Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.742777954Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1564}
128342023-09-22T23:29:48.746Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.742829016Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1564}
128352023-09-22T23:29:48.749Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.742849256Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1564}
128362023-09-22T23:29:48.754ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
128372023-09-22T23:29:48.757Z 0 000-019 80 80 80 123 123 123 F F F
128382023-09-22T23:29:48.760Z 1 020-039 93 93 93 139 139 139 F F F
128392023-09-22T23:29:48.763Z 2 040-059 93 93 93 139 139 139 F F F
128402023-09-22T23:29:48.766Z 3 060-079 96 96 96 146 146 146 F F F
128412023-09-22T23:29:48.769Z 4 080-099 90 90 90 137 137 137 F F F
128422023-09-22T23:29:48.773Z 5 100-119 94 94 94 143 143 143 F F F
128432023-09-22T23:29:48.776Z 6 120-139 85 85 85 128 128 128 F F F
128442023-09-22T23:29:48.779Z 7 140-159 94 94 94 143 143 143 F F F
128452023-09-22T23:29:48.782Z 8 160-179 94 94 94 143 143 143 F F F
128462023-09-22T23:29:48.785Z 9 180-199 95 95 95 144 144 144 F F F
128472023-09-22T23:29:48.788Z 10 200-219 97 97 97 147 147 147 F F F
128482023-09-22T23:29:48.790Z 11 220-239 96 96 96 146 146 146 F F F
128492023-09-22T23:29:48.793Z 12 240-259 93 93 93 139 139 139 F F F
128502023-09-22T23:29:48.797Z 13 260-279 96 96 96 146 146 146 F F F
128512023-09-22T23:29:48.800Z 14 280-299 94 94 94 142 142 142 F F F
128522023-09-22T23:29:48.802Z 15 300-319 94 94 94 143 143 143 F F F
128532023-09-22T23:29:48.805Z 16 320-339 97 97 97 147 147 147 F F F
128542023-09-22T23:29:48.808Z 17 340-359 92 92 92 138 138 138 F F F
128552023-09-22T23:29:48.815Z 18 360-379 90 90 90 137 137 137 F F F
128562023-09-22T23:29:48.826Z 19 380-399 93 93 93 139 139 139 F F F
128572023-09-22T23:29:48.829Z 20 400-419 96 96 96 146 146 146 F F F
128582023-09-22T23:29:48.832Z 21 420-439 96 96 96 146 146 146 F F F
128592023-09-22T23:29:48.835Z 22 440-459 94 94 94 143 143 143 F F F
128602023-09-22T23:29:48.841Z 23 460-479 96 96 96 146 146 146 F F F
128612023-09-22T23:29:48.843Z 24 480-499 96 96 96 146 146 146 F F F
128622023-09-22T23:29:48.847Z 25 500-519 97 97 97 147 147 147 F F F
128632023-09-22T23:29:48.852Z 26 520-539 97 97 97 147 147 147 F F F
128642023-09-22T23:29:48.855Z 27 540-559 85 85 85 128 128 128 F F F
128652023-09-22T23:29:48.858Z 28 560-579 94 94 94 140 140 140 F F F
128662023-09-22T23:29:48.861Z 29 580-599 94 94 94 143 143 143 F F F
128672023-09-22T23:29:48.864ZMax gen: 97, Max flush: 147
128682023-09-22T23:29:48.867Z
128692023-09-22T23:29:48.870ZBegin loop 49
128702023-09-22T23:29:48.873ZDownstairs to restart: 0
128712023-09-22T23:29:48.876Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.792232743Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
128722023-09-22T23:29:48.879Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.792616025Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
128732023-09-22T23:29:48.882Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.792692014Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
128742023-09-22T23:29:48.885Z{"msg":"Crucible stats registered with UUID: a7b393ef-f456-4484-b811-fa84de3befa2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.792769269Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
128752023-09-22T23:29:48.889Z{"msg":"Crucible a7b393ef-f456-4484-b811-fa84de3befa2 has session id: 0d795619-da34-48de-a6ab-fd31886bd583","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.792810668Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
128762023-09-22T23:29:48.892ZCrucible runtime is spawned
128772023-09-22T23:29:48.895ZThe guest has requested activation with gen:98
128782023-09-22T23:29:48.898Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.793010522Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566,"task":"up_listen"}
128792023-09-22T23:29:48.901Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.793079695Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
128802023-09-22T23:29:48.904Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.793137198Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
128812023-09-22T23:29:48.907Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.79321664Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566,"looper":"0"}
128822023-09-22T23:29:48.910Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.79328164Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566,"looper":"1"}
128832023-09-22T23:29:48.913Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.793515476Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566,"looper":"2"}
128842023-09-22T23:29:48.917Z{"msg":"[1] a7b393ef-f456-4484-b811-fa84de3befa2 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.793567265Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566,"looper":"1"}
128852023-09-22T23:29:48.920Z{"msg":"[2] a7b393ef-f456-4484-b811-fa84de3befa2 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.793644722Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566,"looper":"2"}
128862023-09-22T23:29:48.923Z{"msg":"a7b393ef-f456-4484-b811-fa84de3befa2 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.793924943Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
128872023-09-22T23:29:48.926Z{"msg":"Set desired generation to :98","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.794096819Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
128882023-09-22T23:29:48.930Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.794153113Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
128892023-09-22T23:29:48.933Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.794445876Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
128902023-09-22T23:29:48.936Z{"msg":"[1] a7b393ef-f456-4484-b811-fa84de3befa2 (255fea52-8333-4963-a4e7-efd3922a8e77) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.794833998Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
128912023-09-22T23:29:48.939Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.79490993Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
128922023-09-22T23:29:48.942Z{"msg":"[1] client is_active_req TRUE, promote! session 255fea52-8333-4963-a4e7-efd3922a8e77","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.794980422Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
128932023-09-22T23:29:48.945Z{"msg":"[2] a7b393ef-f456-4484-b811-fa84de3befa2 (255fea52-8333-4963-a4e7-efd3922a8e77) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.795051678Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
128942023-09-22T23:29:48.949Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.795184474Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
128952023-09-22T23:29:48.952Z{"msg":"[2] client is_active_req TRUE, promote! session 255fea52-8333-4963-a4e7-efd3922a8e77","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.79524394Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
128962023-09-22T23:29:48.955Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.79539808Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
128972023-09-22T23:29:48.958Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.795465387Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
128982023-09-22T23:29:48.961Z{"msg":"a7b393ef-f456-4484-b811-fa84de3befa2 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.795523151Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
128992023-09-22T23:29:48.964Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.795648313Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129002023-09-22T23:29:48.967Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.795704509Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129012023-09-22T23:29:48.970Z{"msg":"a7b393ef-f456-4484-b811-fa84de3befa2 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.795746393Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129022023-09-22T23:29:48.972Z{"msg":"[1] a7b393ef-f456-4484-b811-fa84de3befa2 (255fea52-8333-4963-a4e7-efd3922a8e77) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.797602532Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129032023-09-22T23:29:48.975Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.797666388Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129042023-09-22T23:29:48.978Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:29:48.797710055Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129052023-09-22T23:29:48.981Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.797732049Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129062023-09-22T23:29:48.984Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.797769389Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129072023-09-22T23:29:48.987Z{"msg":"a7b393ef-f456-4484-b811-fa84de3befa2 New WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.797808805Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129082023-09-22T23:29:48.990Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.797830665Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129092023-09-22T23:29:48.993Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.797866698Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129102023-09-22T23:29:48.996Z{"msg":"[2] a7b393ef-f456-4484-b811-fa84de3befa2 (255fea52-8333-4963-a4e7-efd3922a8e77) New WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.798838209Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129112023-09-22T23:29:48.999Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.798886894Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129122023-09-22T23:29:49.002Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:29:48.798947498Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129132023-09-22T23:29:49.005Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.798990903Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129142023-09-22T23:29:49.008Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.799031362Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129152023-09-22T23:29:49.011Z{"msg":"a7b393ef-f456-4484-b811-fa84de3befa2 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.799070847Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129162023-09-22T23:29:49.014Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.799111465Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129172023-09-22T23:29:49.017Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:48.799131164Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129182023-09-22T23:29:49.795Z{"msg":"[0] a7b393ef-f456-4484-b811-fa84de3befa2 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.795079798Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566,"looper":"0"}
129192023-09-22T23:29:49.798Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.795168216Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129202023-09-22T23:29:49.801Z{"msg":"[0] a7b393ef-f456-4484-b811-fa84de3befa2 (255fea52-8333-4963-a4e7-efd3922a8e77) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.796380429Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129212023-09-22T23:29:49.805Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.796437479Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129222023-09-22T23:29:49.807Z{"msg":"[0] client is_active_req TRUE, promote! session 255fea52-8333-4963-a4e7-efd3922a8e77","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.796480947Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129232023-09-22T23:29:49.810Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.797145235Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129242023-09-22T23:29:49.813Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.797202863Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129252023-09-22T23:29:49.816Z{"msg":"a7b393ef-f456-4484-b811-fa84de3befa2 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.797246722Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129262023-09-22T23:29:49.819Z{"msg":"[0] a7b393ef-f456-4484-b811-fa84de3befa2 (255fea52-8333-4963-a4e7-efd3922a8e77) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.799667155Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129272023-09-22T23:29:49.822Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.799699163Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129282023-09-22T23:29:49.825Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:29:49.799755563Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129292023-09-22T23:29:49.828Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.799776622Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129302023-09-22T23:29:49.831Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.799795471Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129312023-09-22T23:29:49.834Z{"msg":"a7b393ef-f456-4484-b811-fa84de3befa2 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.799834112Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129322023-09-22T23:29:49.837Z{"msg":"[0]R flush_numbers[0..12]: [123, 139, 139, 146, 137, 143, 128, 143, 143, 144, 147, 146]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.799874748Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129332023-09-22T23:29:49.840Z{"msg":"[0]R generation[0..12]: [80, 93, 93, 96, 90, 94, 85, 94, 94, 95, 97, 96]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.799914706Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129342023-09-22T23:29:49.843Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.799955344Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129352023-09-22T23:29:49.846Z{"msg":"[1]R flush_numbers[0..12]: [123, 139, 139, 146, 137, 143, 128, 143, 143, 144, 147, 146]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.800000361Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129362023-09-22T23:29:49.848Z{"msg":"[1]R generation[0..12]: [80, 93, 93, 96, 90, 94, 85, 94, 94, 95, 97, 96]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.800040104Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129372023-09-22T23:29:49.851Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:98
129382023-09-22T23:29:49.854ZWait for a query_work_queue command to finish before sending IO
129392023-09-22T23:29:49.857Z,"time":"2023-09-22T23:29:49.800114706Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129402023-09-22T23:29:49.860Z{"msg":"[2]R flush_numbers[0..12]: [123, 139, 139, 146, 137, 143, 128, 143, 143, 144, 147, 146]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.800352397Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129412023-09-22T23:29:49.863Z{"msg":"[2]R generation[0..12]: [80, 93, 93, 96, 90, 94, 85, 94, 94, 95, 97, 96]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.800402718Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129422023-09-22T23:29:49.866Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.800471363Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129432023-09-22T23:29:49.869Z{"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.800517835Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129442023-09-22T23:29:49.872Z{"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.80054003Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129452023-09-22T23:29:49.874Z{"msg":"Next flush: 148","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.800577237Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129462023-09-22T23:29:49.877Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.800617094Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129472023-09-22T23:29:49.880Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.800672315Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129482023-09-22T23:29:49.893Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.800735235Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129492023-09-22T23:29:49.896Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.800758399Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129502023-09-22T23:29:49.900Z{"msg":"a7b393ef-f456-4484-b811-fa84de3befa2 is now active with session: 255fea52-8333-4963-a4e7-efd3922a8e77","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.800804551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129512023-09-22T23:29:49.903Z{"msg":"a7b393ef-f456-4484-b811-fa84de3befa2 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.80086858Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129522023-09-22T23:29:49.906Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.800912886Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129532023-09-22T23:29:49.909Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.800966226Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129542023-09-22T23:29:49.912Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.801027727Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129552023-09-22T23:29:49.915Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.801070745Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129562023-09-22T23:29:49.918Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
129572023-09-22T23:29:49.921Z,"time":"2023-09-22T23:29:49.801144751Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129582023-09-22T23:29:49.927Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.801219218Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129592023-09-22T23:29:49.930Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.801264596Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129602023-09-22T23:29:49.933Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.801303035Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129612023-09-22T23:29:49.936Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.801341814Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129622023-09-22T23:29:49.939Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.801380037Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129632023-09-22T23:29:49.941Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.801418188Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1566}
129642023-09-22T23:29:49.944ZLoading write count information from file "/tmp/test_repair_verify.data"
129652023-09-22T23:29:49.947ZRun Repair workload
129662023-09-22T23:29:49.950Z01/30 Write block 5 len 36864 data: 5 5 7 7 6 6 6 6 7
129672023-09-22T23:29:49.953Z02/30 Write block 546 len 32768 data: 8 6 10 12 11 11 10 9
129682023-09-22T23:29:49.957Z03/30 Write block 301 len 28672 data: 7 8 9 10 9 9 9
129692023-09-22T23:29:49.959Z04/30 Write block 472 len 8192 data: 6 6
129702023-09-22T23:29:49.962Z05/30 Write block 435 len 40960 data: 10 9 8 7 5 5 8 9 11 11
129712023-09-22T23:29:49.965Z06/30 Flush
129722023-09-22T23:29:49.968Z07/30 Write block 456 len 16384 data: 6 6 6 10
129732023-09-22T23:29:49.971Z08/30 Read block 523 len 8192
129742023-09-22T23:29:49.974Z09/30 Read block 591 len 36864
129752023-09-22T23:29:49.977Z10/30 Write block 546 len 4096 data: 9
129762023-09-22T23:29:49.980Z11/30 Read block 297 len 28672
129772023-09-22T23:29:49.982Z12/30 Read block 87 len 36864
129782023-09-22T23:29:49.985Z13/30 Write block 185 len 12288 data: 6 7 6
129792023-09-22T23:29:49.988Z14/30 Flush
129802023-09-22T23:29:49.991Z15/30 Write block 115 len 20480 data: 5 6 7 6 5
129812023-09-22T23:29:49.997Z16/30 Read block 278 len 24576
129822023-09-22T23:29:50.000Z17/30 Flush
129832023-09-22T23:29:50.003Z18/30 Write block 201 len 20480 data: 7 9 10 9 14
129842023-09-22T23:29:50.006Z19/30 Read block 324 len 24576
129852023-09-22T23:29:50.013Z20/30 Write block 9 len 20480 data: 7 7 7 7 8
129862023-09-22T23:29:50.016Z21/30 Flush
129872023-09-22T23:29:50.019Z22/30 Write block 484 len 40960 data: 11 11 13 10 14 15 14 15 13 13
129882023-09-22T23:29:50.022Z23/30 Write block 146 len 20480 data: 5 5 8 9 9
129892023-09-22T23:29:50.025Z24/30 Write block 282 len 4096 data: 12
129902023-09-22T23:29:50.029Z25/30 Read block 120 len 24576
129912023-09-22T23:29:50.032Z26/30 Write block 327 len 32768 data: 5 8 6 5 7 8 8 10
129922023-09-22T23:29:50.035Z27/30 Write block 503 len 28672 data: 7 8 7 8 8 6 10
129932023-09-22T23:29:50.038Z28/30 Read block 572 len 16384
129942023-09-22T23:29:50.041Z29/30 Read block 42 len 32768
129952023-09-22T23:29:50.053Z30/30 Read block 191 len 40960
129962023-09-22T23:29:50.061Z----------------------------------------------------------------
129972023-09-22T23:29:50.070Z Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30
129982023-09-22T23:29:50.078ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
129992023-09-22T23:29:50.082Z 1 Acked 1000 Write 9 Sent Done Done false
130002023-09-22T23:29:50.085Z 2 Acked 1001 Write 8 Sent Done Done false
130012023-09-22T23:29:50.090Z 3 Acked 1002 Write 7 Sent Done Done false
130022023-09-22T23:29:50.094Z 4 Acked 1003 Write 2 Sent Done Done false
130032023-09-22T23:29:50.098Z 5 Acked 1004 Write 10 Sent Done Done false
130042023-09-22T23:29:50.102Z 6 Acked 1005 Flush 0 Sent Done Done false
130052023-09-22T23:29:50.105Z 7 Acked 1006 Write 4 Sent Done Done false
130062023-09-22T23:29:50.109Z 8 Acked 1007 Read 2 Sent Done Done false
130072023-09-22T23:29:50.113Z 9 Acked 1008 Read 9 Sent Done Done false
130082023-09-22T23:29:50.117Z 10 Acked 1009 Write 1 Sent Done Done false
130092023-09-22T23:29:50.121Z 11 Acked 1010 Read 7 Sent Done Done false
130102023-09-22T23:29:50.124Z 12 Acked 1011 Read 9 Sent Done Done false
130112023-09-22T23:29:50.130Z 13 Acked 1012 Write 3 Sent Done Done false
130122023-09-22T23:29:50.136Z 14 Acked 1013 Flush 0 Sent Done Done false
130132023-09-22T23:29:50.139Z 15 Acked 1014 Write 5 Sent Done Done false
130142023-09-22T23:29:50.143Z 16 Acked 1015 Read 6 Sent Done Done false
130152023-09-22T23:29:50.146Z 17 Acked 1016 Flush 0 Sent Done Done false
130162023-09-22T23:29:50.150Z 18 Acked 1017 Write 5 Sent Done Done false
130172023-09-22T23:29:50.155Z 19 Acked 1018 Read 6 Sent Done Done false
130182023-09-22T23:29:50.161Z 20 Acked 1019 Write 5 Sent Done Done false
130192023-09-22T23:29:50.164Z 21 Acked 1020 Flush 0 Sent Done Done false
130202023-09-22T23:29:50.167Z 22 Acked 1021 Write 10 Sent Done Done false
130212023-09-22T23:29:50.170Z 23 Acked 1022 Write 5 Sent Done Done false
130222023-09-22T23:29:50.173Z 24 Acked 1023 Write 1 Sent Done Done false
130232023-09-22T23:29:50.176Z 25 Acked 1024 Read 6 Sent Done Done false
130242023-09-22T23:29:50.200Z 26 Acked 1025 Write 8 Sent Done Done false
130252023-09-22T23:29:50.203Z 27 Acked 1026 Write 7 Sent Done Done false
130262023-09-22T23:29:50.206Z 28 Acked 1027 Read 4 Sent Done Done false
130272023-09-22T23:29:50.208Z 29 Acked 1028 Read 8 Sent Done Done false
130282023-09-22T23:29:50.211Z 30 Acked 1029 Read 10 Sent Done Done false
130292023-09-22T23:29:50.214Z STATES DS:0 DS:1 DS:2 TOTAL
130302023-09-22T23:29:50.217Z New 0 0 0 0
130312023-09-22T23:29:50.219Z Sent 30 0 0 30
130322023-09-22T23:29:50.222Z Done 0 30 30 60
130332023-09-22T23:29:50.225Z Skipped 0 0 0 0
130342023-09-22T23:29:50.228Z Error 0 0 0 0
130352023-09-22T23:29:50.232ZLast Flush: 0 1020 1020
130362023-09-22T23:29:50.236ZDownstairs last five completed:
130372023-09-22T23:29:50.239ZUpstairs last five completed: 30 29 28 27 26 25
130382023-09-22T23:29:50.242ZWrote out file "/tmp/test_repair_verify.data"
130392023-09-22T23:29:50.245Z
130402023-09-22T23:29:50.248ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
130412023-09-22T23:29:50.250Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.9468269Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1567}
130422023-09-22T23:29:50.253Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.947137581Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1567}
130432023-09-22T23:29:50.256Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.947160972Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1567}
130442023-09-22T23:29:50.259Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.974714217Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1567}
130452023-09-22T23:29:50.261Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.974746884Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1567}
130462023-09-22T23:29:50.264Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.974766202Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1567}
130472023-09-22T23:29:50.267Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.99367288Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1567}
130482023-09-22T23:29:50.274Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.99370401Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1567}
130492023-09-22T23:29:50.284Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:49.993722977Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1567}
130502023-09-22T23:29:50.291ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
130512023-09-22T23:29:50.295ZError: Difference in extent metadata found!
130522023-09-22T23:29:50.298Z 0 000-019 80 98 98 123 151 151 F F F <---
130532023-09-22T23:29:50.301Z 1 020-039 93 93 93 139 139 139 F F F
130542023-09-22T23:29:50.303Z 2 040-059 93 93 93 139 139 139 F F F
130552023-09-22T23:29:50.307Z 3 060-079 96 96 96 146 146 146 F F F
130562023-09-22T23:29:50.310Z 4 080-099 90 90 90 137 137 137 F F F
130572023-09-22T23:29:50.313Z 5 100-119 94 98 98 143 150 150 F F F <---
130582023-09-22T23:29:50.316Z 6 120-139 85 85 85 128 128 128 F F F
130592023-09-22T23:29:50.319Z 7 140-159 94 94 94 143 143 143 F T T <---
130602023-09-22T23:29:50.322Z 8 160-179 94 94 94 143 143 143 F F F
130612023-09-22T23:29:50.325Z 9 180-199 95 98 98 144 149 149 F F F <---
130622023-09-22T23:29:50.328Z 10 200-219 97 98 98 147 151 151 F F F <---
130632023-09-22T23:29:50.331Z 11 220-239 96 96 96 146 146 146 F F F
130642023-09-22T23:29:50.334Z 12 240-259 93 93 93 139 139 139 F F F
130652023-09-22T23:29:50.337Z 13 260-279 96 96 96 146 146 146 F F F
130662023-09-22T23:29:50.340Z 14 280-299 94 94 94 142 142 142 F T T <---
130672023-09-22T23:29:50.343Z 15 300-319 94 98 98 143 148 148 F F F <---
130682023-09-22T23:29:50.346Z 16 320-339 97 97 97 147 147 147 F T T <---
130692023-09-22T23:29:50.348Z 17 340-359 92 92 92 138 138 138 F F F
130702023-09-22T23:29:50.352Z 18 360-379 90 90 90 137 137 137 F F F
130712023-09-22T23:29:50.354Z 19 380-399 93 93 93 139 139 139 F F F
130722023-09-22T23:29:50.357Z 20 400-419 96 96 96 146 146 146 F F F
130732023-09-22T23:29:50.360Z 21 420-439 96 98 98 146 148 148 F F F <---
130742023-09-22T23:29:50.364Z 22 440-459 94 98 98 143 149 149 F F F <---
130752023-09-22T23:29:50.367Z 23 460-479 96 98 98 146 148 148 F F F <---
130762023-09-22T23:29:50.369Z 24 480-499 96 96 96 146 146 146 F T T <---
130772023-09-22T23:29:50.372Z 25 500-519 97 97 97 147 147 147 F T T <---
130782023-09-22T23:29:50.376Z 26 520-539 97 97 97 147 147 147 F F F
130792023-09-22T23:29:50.379Z 27 540-559 85 98 98 128 149 149 F F F <---
130802023-09-22T23:29:50.381Z 28 560-579 94 94 94 140 140 140 F F F
130812023-09-22T23:29:50.385Z 29 580-599 94 94 94 143 143 143 F F F
130822023-09-22T23:29:50.388ZMax gen: 98, Max flush: 151
130832023-09-22T23:29:50.391ZA Difference in extent metadata is expected here
130842023-09-22T23:29:50.394ZOn loop 49
130852023-09-22T23:29:50.397Z
130862023-09-22T23:29:50.400ZVerifying data now
130872023-09-22T23:29:56.491ZNo change after verify
130882023-09-22T23:29:56.494ZLoop: 49 Downstairs dump after verify (and repair):
130892023-09-22T23:29:56.510Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.510444105Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1572}
130902023-09-22T23:29:56.513Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.510788241Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1572}
130912023-09-22T23:29:56.516Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.510813281Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1572}
130922023-09-22T23:29:56.530Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.530116293Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1572}
130932023-09-22T23:29:56.533Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.530175941Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1572}
130942023-09-22T23:29:56.535Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.530196205Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1572}
130952023-09-22T23:29:56.541Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.541303669Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1572}
130962023-09-22T23:29:56.544Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.541358267Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1572}
130972023-09-22T23:29:56.546Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.541378672Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1572}
130982023-09-22T23:29:56.552ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
130992023-09-22T23:29:56.555Z 0 000-019 98 98 98 151 151 151 F F F
131002023-09-22T23:29:56.558Z 1 020-039 93 93 93 139 139 139 F F F
131012023-09-22T23:29:56.560Z 2 040-059 93 93 93 139 139 139 F F F
131022023-09-22T23:29:56.563Z 3 060-079 96 96 96 146 146 146 F F F
131032023-09-22T23:29:56.566Z 4 080-099 90 90 90 137 137 137 F F F
131042023-09-22T23:29:56.569Z 5 100-119 98 98 98 150 150 150 F F F
131052023-09-22T23:29:56.573Z 6 120-139 85 85 85 128 128 128 F F F
131062023-09-22T23:29:56.576Z 7 140-159 99 99 99 152 152 152 F F F
131072023-09-22T23:29:56.579Z 8 160-179 94 94 94 143 143 143 F F F
131082023-09-22T23:29:56.582Z 9 180-199 98 98 98 149 149 149 F F F
131092023-09-22T23:29:56.585Z 10 200-219 98 98 98 151 151 151 F F F
131102023-09-22T23:29:56.587Z 11 220-239 96 96 96 146 146 146 F F F
131112023-09-22T23:29:56.590Z 12 240-259 93 93 93 139 139 139 F F F
131122023-09-22T23:29:56.593Z 13 260-279 96 96 96 146 146 146 F F F
131132023-09-22T23:29:56.596Z 14 280-299 99 99 99 152 152 152 F F F
131142023-09-22T23:29:56.599Z 15 300-319 98 98 98 148 148 148 F F F
131152023-09-22T23:29:56.602Z 16 320-339 99 99 99 152 152 152 F F F
131162023-09-22T23:29:56.604Z 17 340-359 92 92 92 138 138 138 F F F
131172023-09-22T23:29:56.607Z 18 360-379 90 90 90 137 137 137 F F F
131182023-09-22T23:29:56.610Z 19 380-399 93 93 93 139 139 139 F F F
131192023-09-22T23:29:56.612Z 20 400-419 96 96 96 146 146 146 F F F
131202023-09-22T23:29:56.615Z 21 420-439 98 98 98 148 148 148 F F F
131212023-09-22T23:29:56.618Z 22 440-459 98 98 98 149 149 149 F F F
131222023-09-22T23:29:56.621Z 23 460-479 98 98 98 148 148 148 F F F
131232023-09-22T23:29:56.623Z 24 480-499 99 99 99 152 152 152 F F F
131242023-09-22T23:29:56.626Z 25 500-519 99 99 99 152 152 152 F F F
131252023-09-22T23:29:56.629Z 26 520-539 97 97 97 147 147 147 F F F
131262023-09-22T23:29:56.632Z 27 540-559 98 98 98 149 149 149 F F F
131272023-09-22T23:29:56.635Z 28 560-579 94 94 94 140 140 140 F F F
131282023-09-22T23:29:56.638Z 29 580-599 94 94 94 143 143 143 F F F
131292023-09-22T23:29:56.641ZMax gen: 99, Max flush: 152
131302023-09-22T23:29:56.643Z
131312023-09-22T23:29:56.647ZBegin loop 50
131322023-09-22T23:29:56.650ZDownstairs to restart: 1
131332023-09-22T23:29:56.652Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.593606233Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131342023-09-22T23:29:56.655Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.594005876Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131352023-09-22T23:29:56.658Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.59406528Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131362023-09-22T23:29:56.660Z{"msg":"Crucible stats registered with UUID: d98dc2c7-847a-45f7-a49b-221094e24c14","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.594108841Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131372023-09-22T23:29:56.663Z{"msg":"Crucible d98dc2c7-847a-45f7-a49b-221094e24c14 has session id: a4f67617-f005-45d1-975e-b26200d9cd59","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.59417194Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131382023-09-22T23:29:56.666ZCrucible runtime is spawned
131392023-09-22T23:29:56.669ZThe guest has requested activation with gen:100
131402023-09-22T23:29:56.671Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.594454319Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574,"task":"up_listen"}
131412023-09-22T23:29:56.674Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.594510468Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131422023-09-22T23:29:56.677Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.594551119Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131432023-09-22T23:29:56.680Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.594590051Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574,"looper":"2"}
131442023-09-22T23:29:56.683Z{"msg":"[2] d98dc2c7-847a-45f7-a49b-221094e24c14 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.594809625Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574,"looper":"2"}
131452023-09-22T23:29:56.685Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.594889031Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574,"looper":"1"}
131462023-09-22T23:29:56.688Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.595055036Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574,"looper":"0"}
131472023-09-22T23:29:56.691Z{"msg":"[0] d98dc2c7-847a-45f7-a49b-221094e24c14 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.595243396Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574,"looper":"0"}
131482023-09-22T23:29:56.695Z{"msg":"d98dc2c7-847a-45f7-a49b-221094e24c14 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.595440788Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131492023-09-22T23:29:56.700Z{"msg":"Set desired generation to :100","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.595495254Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131502023-09-22T23:29:56.703Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.595628388Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131512023-09-22T23:29:56.706Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.595929605Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131522023-09-22T23:29:56.709Z{"msg":"[2] d98dc2c7-847a-45f7-a49b-221094e24c14 (6f18fc10-e750-4775-9989-299f8f00aef1) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.596348337Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131532023-09-22T23:29:56.711Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.596425752Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131542023-09-22T23:29:56.714Z{"msg":"[2] client is_active_req TRUE, promote! session 6f18fc10-e750-4775-9989-299f8f00aef1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.596505811Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131552023-09-22T23:29:56.717Z{"msg":"[0] d98dc2c7-847a-45f7-a49b-221094e24c14 (6f18fc10-e750-4775-9989-299f8f00aef1) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.596578566Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131562023-09-22T23:29:56.720Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.596641531Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131572023-09-22T23:29:56.723Z{"msg":"[0] client is_active_req TRUE, promote! session 6f18fc10-e750-4775-9989-299f8f00aef1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.596687931Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131582023-09-22T23:29:56.726Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.596986045Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131592023-09-22T23:29:56.729Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.59706262Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131602023-09-22T23:29:56.733Z{"msg":"d98dc2c7-847a-45f7-a49b-221094e24c14 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.597136363Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131612023-09-22T23:29:56.736Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.597218194Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131622023-09-22T23:29:56.739Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.597296328Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131632023-09-22T23:29:56.741Z{"msg":"d98dc2c7-847a-45f7-a49b-221094e24c14 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.597368258Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131642023-09-22T23:29:56.744Z{"msg":"[2] d98dc2c7-847a-45f7-a49b-221094e24c14 (6f18fc10-e750-4775-9989-299f8f00aef1) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.599467244Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131652023-09-22T23:29:56.748Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.599542005Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131662023-09-22T23:29:56.751Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:29:56.599609047Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131672023-09-22T23:29:56.754Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.599661678Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131682023-09-22T23:29:56.757Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.599702974Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131692023-09-22T23:29:56.760Z{"msg":"d98dc2c7-847a-45f7-a49b-221094e24c14 WaitActive New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.599744402Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131702023-09-22T23:29:56.763Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.59978274Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131712023-09-22T23:29:56.766Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.599821101Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131722023-09-22T23:29:56.769Z{"msg":"[0] d98dc2c7-847a-45f7-a49b-221094e24c14 (6f18fc10-e750-4775-9989-299f8f00aef1) WaitActive New WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.599862546Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131732023-09-22T23:29:56.772Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.599901448Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131742023-09-22T23:29:56.775Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:29:56.599940119Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131752023-09-22T23:29:56.778Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.599977899Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131762023-09-22T23:29:56.781Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.600016516Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131772023-09-22T23:29:56.783Z{"msg":"d98dc2c7-847a-45f7-a49b-221094e24c14 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.600054742Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131782023-09-22T23:29:56.787Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.600094283Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131792023-09-22T23:29:56.789Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:56.600131531Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131802023-09-22T23:29:57.596Z{"msg":"[1] d98dc2c7-847a-45f7-a49b-221094e24c14 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.595825669Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574,"looper":"1"}
131812023-09-22T23:29:57.600Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.595909251Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131822023-09-22T23:29:57.605Z{"msg":"[1] d98dc2c7-847a-45f7-a49b-221094e24c14 (6f18fc10-e750-4775-9989-299f8f00aef1) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.59699833Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131832023-09-22T23:29:57.610Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.597062049Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131842023-09-22T23:29:57.613Z{"msg":"[1] client is_active_req TRUE, promote! session 6f18fc10-e750-4775-9989-299f8f00aef1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.597087566Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131852023-09-22T23:29:57.617Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.598010241Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131862023-09-22T23:29:57.620Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.598071438Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131872023-09-22T23:29:57.624Z{"msg":"d98dc2c7-847a-45f7-a49b-221094e24c14 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.5981107Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131882023-09-22T23:29:57.627Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.600811711Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131892023-09-22T23:29:57.630Z{"msg":"[1] d98dc2c7-847a-45f7-a49b-221094e24c14 (6f18fc10-e750-4775-9989-299f8f00aef1) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.601047965Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131902023-09-22T23:29:57.633Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.601168957Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131912023-09-22T23:29:57.636Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:29:57.601283558Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131922023-09-22T23:29:57.639Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.601353318Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131932023-09-22T23:29:57.642Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.601498928Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131942023-09-22T23:29:57.645Z{"msg":"d98dc2c7-847a-45f7-a49b-221094e24c14 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:100
131952023-09-22T23:29:57.648ZWait for a query_work_queue command to finish before sending IO
131962023-09-22T23:29:57.652Z,"time":"2023-09-22T23:29:57.601605046Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131972023-09-22T23:29:57.656Z{"msg":"[0]R flush_numbers[0..12]: [151, 139, 139, 146, 137, 150, 128, 152, 143, 149, 151, 146]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.601776754Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131982023-09-22T23:29:57.660Z{"msg":"[0]R generation[0..12]: [98, 93, 93, 96, 90, 98, 85, 99, 94, 98, 98, 96]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.601823571Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
131992023-09-22T23:29:57.668Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.601915403Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132002023-09-22T23:29:57.671Z{"msg":"[1]R flush_numbers[0..12]: [151, 139, 139, 146, 137, 150, 128, 152, 143, 149, 151, 146]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.601958885Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132012023-09-22T23:29:57.675Z{"msg":"[1]R generation[0..12]: [98, 93, 93, 96, 90, 98, 85, 99, 94, 98, 98, 96]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.601999799Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132022023-09-22T23:29:57.684Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.602047391Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132032023-09-22T23:29:57.688Z{"msg":"[2]R flush_numbers[0..12]: [151, 139, 139, 146, 137, 150, 128, 152, 143, 149, 151, 146]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.602086598Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132042023-09-22T23:29:57.691Z{"msg":"[2]R generation[0..12]: [98, 93, 93, 96, 90, 98, 85, 99, 94, 98, 98, 96]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.602126301Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132052023-09-22T23:29:57.694Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.602166545Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132062023-09-22T23:29:57.698Z{"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.602219674Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132072023-09-22T23:29:57.702Z{"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.602262469Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132082023-09-22T23:29:57.707Z{"msg":"Next flush: 153","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.602305612Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132092023-09-22T23:29:57.713Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.602343558Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132102023-09-22T23:29:57.718Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.602381705Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132112023-09-22T23:29:57.724Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.602401834Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132122023-09-22T23:29:57.735Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.60245288Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132132023-09-22T23:29:57.742Z{"msg":"d98dc2c7-847a-45f7-a49b-221094e24c14 is now active with session: 6f18fc10-e750-4775-9989-299f8f00aef1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.602509435Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132142023-09-22T23:29:57.746Z{"msg":"d98dc2c7-847a-45f7-a49b-221094e24c14 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.602551745Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132152023-09-22T23:29:57.751Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.602595979Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132162023-09-22T23:29:57.757Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.602647797Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132172023-09-22T23:29:57.762Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.602703002Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132182023-09-22T23:29:57.767Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
132192023-09-22T23:29:57.772Z,"time":"2023-09-22T23:29:57.602757421Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132202023-09-22T23:29:57.787Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.602914828Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132212023-09-22T23:29:57.790Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.603011117Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132222023-09-22T23:29:57.796Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.603056239Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132232023-09-22T23:29:57.800Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.603094761Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132242023-09-22T23:29:57.803Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.603146797Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132252023-09-22T23:29:57.806Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.603168056Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132262023-09-22T23:29:57.810Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.603218953Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1574}
132272023-09-22T23:29:57.813ZLoading write count information from file "/tmp/test_repair_verify.data"
132282023-09-22T23:29:57.816ZRun Repair workload
132292023-09-22T23:29:57.819Z01/30 Write block 130 len 40960 data: 7 6 10 9 9 8 9 9 9 9
132302023-09-22T23:29:57.822Z02/30 Read block 37 len 4096
132312023-09-22T23:29:57.825Z03/30 Write block 571 len 4096 data: 7
132322023-09-22T23:29:57.828Z04/30 Write block 387 len 24576 data: 8 8 7 7 6 7
132332023-09-22T23:29:57.831Z05/30 Write block 95 len 16384 data: 11 10 9 10
132342023-09-22T23:29:57.835Z06/30 Write block 159 len 16384 data: 11 10 10 10
132352023-09-22T23:29:57.838Z07/30 Write block 172 len 36864 data: 10 10 10 11 11 10 10 8 6
132362023-09-22T23:29:57.841Z08/30 Read block 466 len 36864
132372023-09-22T23:29:57.844Z09/30 Write block 552 len 36864 data: 11 10 5 5 6 5 6 7 7
132382023-09-22T23:29:57.847Z10/30 Write block 66 len 12288 data: 18 17 17
132392023-09-22T23:29:57.850Z11/30 Read block 207 len 36864
132402023-09-22T23:29:57.853Z12/30 Write block 411 len 20480 data: 8 5 7 9 9
132412023-09-22T23:29:57.856Z13/30 Read block 84 len 16384
132422023-09-22T23:29:57.859Z14/30 Write block 17 len 8192 data: 6 7
132432023-09-22T23:29:57.862Z15/30 Read block 278 len 32768
132442023-09-22T23:29:57.866Z16/30 Write block 25 len 12288 data: 6 7 8
132452023-09-22T23:29:57.869Z17/30 Read block 476 len 20480
132462023-09-22T23:29:57.872Z18/30 Read block 422 len 28672
132472023-09-22T23:29:57.879Z19/30 Write block 308 len 28672 data: 8 6 4 3 4 6 8
132482023-09-22T23:29:57.884Z20/30 Read block 81 len 20480
132492023-09-22T23:29:57.887Z21/30 Write block 385 len 28672 data: 7 6 9 9 8 8 7
132502023-09-22T23:29:57.890Z22/30 Read block 408 len 36864
132512023-09-22T23:29:57.893Z23/30 Write block 234 len 12288 data: 4 5 6
132522023-09-22T23:29:57.897Z24/30 Read block 544 len 12288
132532023-09-22T23:29:57.900Z25/30 Read block 546 len 28672
132542023-09-22T23:29:57.905Z26/30 Flush
132552023-09-22T23:29:57.909Z27/30 Write block 277 len 24576 data: 13 16 16 15 13 13
132562023-09-22T23:29:57.915Z28/30 Read block 34 len 24576
132572023-09-22T23:29:57.919Z29/30 Write block 394 len 28672 data: 7 9 9 9 10 8 11
132582023-09-22T23:29:57.924Z30/30 Write block 25 len 16384 data: 7 8 9 6
132592023-09-22T23:29:57.928Z----------------------------------------------------------------
132602023-09-22T23:29:57.932Z Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30
132612023-09-22T23:29:57.936ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
132622023-09-22T23:29:57.940Z 1 Acked 1000 Write 10 Done Sent Done false
132632023-09-22T23:29:57.944Z 2 Acked 1001 Read 1 Done Sent Done false
132642023-09-22T23:29:57.948Z 3 Acked 1002 Write 1 Done Sent Done false
132652023-09-22T23:29:57.953Z 4 Acked 1003 Write 6 Done Sent Done false
132662023-09-22T23:29:57.958Z 5 Acked 1004 Write 4 Done Sent Done false
132672023-09-22T23:29:57.963Z 6 Acked 1005 Write 4 Done Sent Done false
132682023-09-22T23:29:57.969Z 7 Acked 1006 Write 9 Done Sent Done false
132692023-09-22T23:29:57.974Z 8 Acked 1007 Read 9 Done Sent Done false
132702023-09-22T23:29:57.977Z 9 Acked 1008 Write 9 Done Sent Done false
132712023-09-22T23:29:57.980Z 10 Acked 1009 Write 3 Done Sent Done false
132722023-09-22T23:29:57.984Z 11 Acked 1010 Read 9 Done Sent Done false
132732023-09-22T23:29:57.987Z 12 Acked 1011 Write 5 Done Sent Done false
132742023-09-22T23:29:57.990Z 13 Acked 1012 Read 4 Done Sent Done false
132752023-09-22T23:29:57.994Z 14 Acked 1013 Write 2 Done Sent Done false
132762023-09-22T23:29:57.998Z 15 Acked 1014 Read 8 Done Sent Done false
132772023-09-22T23:29:58.002Z 16 Acked 1015 Write 3 Done Sent Done false
132782023-09-22T23:29:58.005Z 17 Acked 1016 Read 5 Done Sent Done false
132792023-09-22T23:29:58.008Z 18 Acked 1017 Read 7 Done Sent Done false
132802023-09-22T23:29:58.012Z 19 Acked 1018 Write 7 Done Sent Done false
132812023-09-22T23:29:58.016Z 20 Acked 1019 Read 5 Done Sent Done false
132822023-09-22T23:29:58.019Z 21 Acked 1020 Write 7 Done Sent Done false
132832023-09-22T23:29:58.022Z 22 Acked 1021 Read 9 Done Sent Done false
132842023-09-22T23:29:58.026Z 23 Acked 1022 Write 3 Done Sent Done false
132852023-09-22T23:29:58.029Z 24 Acked 1023 Read 3 Done Sent Done false
132862023-09-22T23:29:58.039Z 25 Acked 1024 Read 7 Done Sent Done false
132872023-09-22T23:29:58.047Z 26 Acked 1025 Flush 0 Done Sent Done false
132882023-09-22T23:29:58.050Z 27 Acked 1026 Write 6 Done Sent Done false
132892023-09-22T23:29:58.056Z 28 Acked 1027 Read 6 Done Sent Done false
132902023-09-22T23:29:58.059Z 29 Acked 1028 Write 7 Sent Sent Sent false
132912023-09-22T23:29:58.062Z 30 Acked 1029 Write 4 Sent Sent Sent false
132922023-09-22T23:29:58.065ZError: Difference in extent metadata found!
132932023-09-22T23:29:58.069Z STATES DS:0 DS:1 DS:2 TOTAL
132942023-09-22T23:29:58.072Z New 0 0 0 0
132952023-09-22T23:29:58.075Z Sent 2 30 2 34
132962023-09-22T23:29:58.078Z Done 28 0 28 56
132972023-09-22T23:29:58.082Z Skipped 0 0 0 0
132982023-09-22T23:29:58.085Z Error 0 0 0 0
132992023-09-22T23:29:58.089ZLast Flush: 1025 0 1025
133002023-09-22T23:29:58.092ZDownstairs last five completed:
133012023-09-22T23:29:58.095ZUpstairs last five completed: 30 29 28 27 26 25
133022023-09-22T23:29:58.098ZWrote out file "/tmp/test_repair_verify.data"
133032023-09-22T23:29:58.101Z
133042023-09-22T23:29:58.104ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
133052023-09-22T23:29:58.106Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.748204217Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1575}
133062023-09-22T23:29:58.109Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.748565141Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1575}
133072023-09-22T23:29:58.112Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.748588498Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1575}
133082023-09-22T23:29:58.116Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.776114046Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1575}
133092023-09-22T23:29:58.119Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.776149244Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1575}
133102023-09-22T23:29:58.122Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.776168343Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1575}
133112023-09-22T23:29:58.125Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.791620638Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1575}
133122023-09-22T23:29:58.128Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.791651028Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1575}
133132023-09-22T23:29:58.132Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:29:57.791669942Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1575}
133142023-09-22T23:29:58.135ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
133152023-09-22T23:29:58.138Z 0 000-019 100 98 100 153 151 153 F F F <---
133162023-09-22T23:29:58.141Z 1 020-039 100 93 100 153 139 153 T F T <---
133172023-09-22T23:29:58.144Z 2 040-059 93 93 93 139 139 139 F F F
133182023-09-22T23:29:58.147Z 3 060-079 100 96 100 153 146 153 F F F <---
133192023-09-22T23:29:58.150Z 4 080-099 100 90 100 153 137 153 F F F <---
133202023-09-22T23:29:58.153Z 5 100-119 98 98 98 150 150 150 F F F
133212023-09-22T23:29:58.156Z 6 120-139 100 85 100 153 128 153 F F F <---
133222023-09-22T23:29:58.159Z 7 140-159 100 99 100 153 152 153 F F F <---
133232023-09-22T23:29:58.162Z 8 160-179 100 94 100 153 143 153 F F F <---
133242023-09-22T23:29:58.165Z 9 180-199 100 98 100 153 149 153 F F F <---
133252023-09-22T23:29:58.168Z 10 200-219 98 98 98 151 151 151 F F F
133262023-09-22T23:29:58.171Z 11 220-239 100 96 100 153 146 153 F F F <---
133272023-09-22T23:29:58.174Z 12 240-259 93 93 93 139 139 139 F F F
133282023-09-22T23:29:58.176Z 13 260-279 96 96 96 146 146 146 T F T <---
133292023-09-22T23:29:58.179Z 14 280-299 99 99 99 152 152 152 T F T <---
133302023-09-22T23:29:58.182Z 15 300-319 100 98 100 153 148 153 F F F <---
133312023-09-22T23:29:58.185Z 16 320-339 99 99 99 152 152 152 F F F
133322023-09-22T23:29:58.188Z 17 340-359 92 92 92 138 138 138 F F F
133332023-09-22T23:29:58.191Z 18 360-379 90 90 90 137 137 137 F F F
133342023-09-22T23:29:58.194Z 19 380-399 100 93 100 153 139 153 T F T <---
133352023-09-22T23:29:58.197Z 20 400-419 100 96 100 153 146 153 T F T <---
133362023-09-22T23:29:58.204Z 21 420-439 98 98 98 148 148 148 F F F
133372023-09-22T23:29:58.618Z 22 440-459 98 98 98 149 149 149 F F F
133382023-09-22T23:29:58.625Z 23 460-479 98 98 98 148 148 148 F F F
133392023-09-22T23:29:58.628Z 24 480-499 99 99 99 152 152 152 F F F
133402023-09-22T23:29:58.634Z 25 500-519 99 99 99 152 152 152 F F F
133412023-09-22T23:29:58.639Z 26 520-539 97 97 97 147 147 147 F F F
133422023-09-22T23:29:58.644Z 27 540-559 100 98 100 153 149 153 F F F <---
133432023-09-22T23:29:58.648Z 28 560-579 100 94 100 153 140 153 F F F <---
133442023-09-22T23:29:58.651Z 29 580-599 94 94 94 143 143 143 F F F
133452023-09-22T23:29:58.654ZMax gen: 100, Max flush: 153
133462023-09-22T23:29:58.658ZA Difference in extent metadata is expected here
133472023-09-22T23:29:58.661ZOn loop 50
133482023-09-22T23:29:58.664Z
133492023-09-22T23:29:58.667ZVerifying data now
133502023-09-22T23:30:04.510ZNo change after verify
133512023-09-22T23:30:04.513ZLoop: 50 Downstairs dump after verify (and repair):
133522023-09-22T23:30:04.517Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.50695844Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1580}
133532023-09-22T23:30:04.520Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.5072877Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1580}
133542023-09-22T23:30:04.523Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.507331933Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1580}
133552023-09-22T23:30:04.526Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.525922132Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1580}
133562023-09-22T23:30:04.529Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.525975709Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1580}
133572023-09-22T23:30:04.533Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.526014101Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1580}
133582023-09-22T23:30:04.536Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.536940726Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1580}
133592023-09-22T23:30:04.540Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.536991729Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1580}
133602023-09-22T23:30:04.544Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.537012095Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1580}
133612023-09-22T23:30:04.547ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
133622023-09-22T23:30:04.551Z 0 000-019 100 100 100 153 153 153 F F F
133632023-09-22T23:30:04.558Z 1 020-039 101 101 101 154 154 154 F F F
133642023-09-22T23:30:04.563Z 2 040-059 93 93 93 139 139 139 F F F
133652023-09-22T23:30:04.567Z 3 060-079 100 100 100 153 153 153 F F F
133662023-09-22T23:30:04.571Z 4 080-099 100 100 100 153 153 153 F F F
133672023-09-22T23:30:04.574Z 5 100-119 98 98 98 150 150 150 F F F
133682023-09-22T23:30:04.578Z 6 120-139 100 100 100 153 153 153 F F F
133692023-09-22T23:30:04.581Z 7 140-159 100 100 100 153 153 153 F F F
133702023-09-22T23:30:04.585Z 8 160-179 100 100 100 153 153 153 F F F
133712023-09-22T23:30:04.589Z 9 180-199 100 100 100 153 153 153 F F F
133722023-09-22T23:30:04.593Z 10 200-219 98 98 98 151 151 151 F F F
133732023-09-22T23:30:04.596Z 11 220-239 100 100 100 153 153 153 F F F
133742023-09-22T23:30:04.599Z 12 240-259 93 93 93 139 139 139 F F F
133752023-09-22T23:30:04.603Z 13 260-279 101 101 101 154 154 154 F F F
133762023-09-22T23:30:04.607Z 14 280-299 101 101 101 154 154 154 F F F
133772023-09-22T23:30:04.610Z 15 300-319 100 100 100 153 153 153 F F F
133782023-09-22T23:30:04.614Z 16 320-339 99 99 99 152 152 152 F F F
133792023-09-22T23:30:04.617Z 17 340-359 92 92 92 138 138 138 F F F
133802023-09-22T23:30:04.621Z 18 360-379 90 90 90 137 137 137 F F F
133812023-09-22T23:30:04.624Z 19 380-399 101 101 101 154 154 154 F F F
133822023-09-22T23:30:04.627Z 20 400-419 101 101 101 154 154 154 F F F
133832023-09-22T23:30:04.630Z 21 420-439 98 98 98 148 148 148 F F F
133842023-09-22T23:30:04.633Z 22 440-459 98 98 98 149 149 149 F F F
133852023-09-22T23:30:04.638Z 23 460-479 98 98 98 148 148 148 F F F
133862023-09-22T23:30:04.641Z 24 480-499 99 99 99 152 152 152 F F F
133872023-09-22T23:30:04.643Z 25 500-519 99 99 99 152 152 152 F F F
133882023-09-22T23:30:04.648Z 26 520-539 97 97 97 147 147 147 F F F
133892023-09-22T23:30:04.651Z 27 540-559 100 100 100 153 153 153 F F F
133902023-09-22T23:30:04.654Z 28 560-579 100 100 100 153 153 153 F F F
133912023-09-22T23:30:04.658Z 29 580-599 94 94 94 143 143 143 F F F
133922023-09-22T23:30:04.662ZMax gen: 101, Max flush: 154
133932023-09-22T23:30:04.665Z
133942023-09-22T23:30:04.668ZBegin loop 51
133952023-09-22T23:30:04.671ZDownstairs to restart: 2
133962023-09-22T23:30:04.676Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.587320525Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
133972023-09-22T23:30:04.680Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.587692064Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
133982023-09-22T23:30:04.683Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.587746438Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
133992023-09-22T23:30:04.687Z{"msg":"Crucible stats registered with UUID: e71f97dc-3db5-4980-9004-9e02f5337158","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.58781319Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134002023-09-22T23:30:04.690Z{"msg":"Crucible e71f97dc-3db5-4980-9004-9e02f5337158 has session id: 8b08dc17-7272-4337-9009-bf188b1f73a5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.587891921Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134012023-09-22T23:30:04.693ZCrucible runtime is spawned
134022023-09-22T23:30:04.696Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.588078294Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582,"task":"up_listen"}
134032023-09-22T23:30:04.700Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30The guest has requested activation with gen:102
134042023-09-22T23:30:04.703Z,"time":"2023-09-22T23:30:04.58818092Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134052023-09-22T23:30:04.707Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.588297808Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134062023-09-22T23:30:04.716Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.58835242Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582,"looper":"0"}
134072023-09-22T23:30:04.727Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.588590009Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582,"looper":"1"}
134082023-09-22T23:30:04.731Z{"msg":"[0] e71f97dc-3db5-4980-9004-9e02f5337158 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.588680123Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582,"looper":"0"}
134092023-09-22T23:30:04.735Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.588764882Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582,"looper":"2"}
134102023-09-22T23:30:04.739Z{"msg":"[1] e71f97dc-3db5-4980-9004-9e02f5337158 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.588839117Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582,"looper":"1"}
134112023-09-22T23:30:04.743Z{"msg":"e71f97dc-3db5-4980-9004-9e02f5337158 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.589120335Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134122023-09-22T23:30:04.747Z{"msg":"Set desired generation to :102","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.589180086Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134132023-09-22T23:30:04.750Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.589224958Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134142023-09-22T23:30:04.753Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.589561049Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134152023-09-22T23:30:04.757Z{"msg":"[0] e71f97dc-3db5-4980-9004-9e02f5337158 (b3540714-57b4-4031-acbd-3d74af480751) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.589923959Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134162023-09-22T23:30:04.760Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.589995729Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134172023-09-22T23:30:04.763Z{"msg":"[0] client is_active_req TRUE, promote! session b3540714-57b4-4031-acbd-3d74af480751","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.590067174Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134182023-09-22T23:30:04.767Z{"msg":"[1] e71f97dc-3db5-4980-9004-9e02f5337158 (b3540714-57b4-4031-acbd-3d74af480751) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.590140255Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134192023-09-22T23:30:04.770Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.590207226Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134202023-09-22T23:30:04.773Z{"msg":"[1] client is_active_req TRUE, promote! session b3540714-57b4-4031-acbd-3d74af480751","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.590272114Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134212023-09-22T23:30:04.777Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.590655665Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134222023-09-22T23:30:04.779Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.590727643Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134232023-09-22T23:30:04.782Z{"msg":"e71f97dc-3db5-4980-9004-9e02f5337158 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.590783643Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134242023-09-22T23:30:04.786Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.590886223Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134252023-09-22T23:30:04.789Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.590959004Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134262023-09-22T23:30:04.792Z{"msg":"e71f97dc-3db5-4980-9004-9e02f5337158 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.591003189Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134272023-09-22T23:30:04.795Z{"msg":"[0] e71f97dc-3db5-4980-9004-9e02f5337158 (b3540714-57b4-4031-acbd-3d74af480751) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.593112552Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134282023-09-22T23:30:04.798Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.593171363Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134292023-09-22T23:30:04.802Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:30:04.593214166Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134302023-09-22T23:30:04.805Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.593253888Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134312023-09-22T23:30:04.808Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.59329328Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134322023-09-22T23:30:04.811Z{"msg":"e71f97dc-3db5-4980-9004-9e02f5337158 WaitQuorum WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.593332738Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134332023-09-22T23:30:04.815Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.593371217Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134342023-09-22T23:30:04.818Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.593415782Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134352023-09-22T23:30:04.821Z{"msg":"[1] e71f97dc-3db5-4980-9004-9e02f5337158 (b3540714-57b4-4031-acbd-3d74af480751) WaitQuorum WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.594301126Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134362023-09-22T23:30:04.824Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.594351345Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134372023-09-22T23:30:04.827Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:30:04.594391527Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134382023-09-22T23:30:04.831Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.594429956Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134392023-09-22T23:30:04.834Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.594450487Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134402023-09-22T23:30:04.837Z{"msg":"e71f97dc-3db5-4980-9004-9e02f5337158 WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.594468877Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134412023-09-22T23:30:04.840Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.59448672Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134422023-09-22T23:30:04.843Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:04.59450412Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134432023-09-22T23:30:05.591Z{"msg":"[2] e71f97dc-3db5-4980-9004-9e02f5337158 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.5904488Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582,"looper":"2"}
134442023-09-22T23:30:05.594Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.590540384Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134452023-09-22T23:30:05.598Z{"msg":"[2] e71f97dc-3db5-4980-9004-9e02f5337158 (b3540714-57b4-4031-acbd-3d74af480751) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.59160726Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134462023-09-22T23:30:05.602Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.59166225Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134472023-09-22T23:30:05.609Z{"msg":"[2] client is_active_req TRUE, promote! session b3540714-57b4-4031-acbd-3d74af480751","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.591705762Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134482023-09-22T23:30:05.614Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.59226395Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134492023-09-22T23:30:05.620Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.592319669Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134502023-09-22T23:30:05.625Z{"msg":"e71f97dc-3db5-4980-9004-9e02f5337158 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.592362569Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134512023-09-22T23:30:05.631Z{"msg":"[2] e71f97dc-3db5-4980-9004-9e02f5337158 (b3540714-57b4-4031-acbd-3d74af480751) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.594757105Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134522023-09-22T23:30:05.637Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.594811472Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134532023-09-22T23:30:05.646Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:30:05.594867891Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134542023-09-22T23:30:05.654Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.594919093Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134552023-09-22T23:30:05.657Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.594959652Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134562023-09-22T23:30:05.661Z{"msg":"e71f97dc-3db5-4980-9004-9e02f5337158 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.594999102Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134572023-09-22T23:30:05.665Z{"msg":"[0]R flush_numbers[0..12]: [153, 154, 139, 153, 153, 150, 153, 153, 153, 153, 151, 153]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.595039966Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134582023-09-22T23:30:05.668Z{"msg":"[0]R generation[0..12]: [100, 101, 93, 100, 100, 98, 100, 100, 100, 100, 98, 100]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.595090112Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134592023-09-22T23:30:05.671Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.595130605Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134602023-09-22T23:30:05.675Z{"msg":"The guest has finished waiting for activation with:102
134612023-09-22T23:30:05.678Z[1]R flush_numbers[0..12]: [153, 154, 139, 153, 153, 150, 153, 153, 153, 153, 151, 153]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
134622023-09-22T23:30:05.681Z,"time":"2023-09-22T23:30:05.595158571Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134632023-09-22T23:30:05.685Z{"msg":"[1]R generation[0..12]: [100, 101, 93, 100, 100, 98, 100, 100, 100, 100, 98, 100]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.595178672Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134642023-09-22T23:30:05.688Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.595222321Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134652023-09-22T23:30:05.693Z{"msg":"[2]R flush_numbers[0..12]: [153, 154, 139, 153, 153, 150, 153, 153, 153, 153, 151, 153]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.595263033Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134662023-09-22T23:30:05.696Z{"msg":"[2]R generation[0..12]: [100, 101, 93, 100, 100, 98, 100, 100, 100, 100, 98, 100]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.595303372Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134672023-09-22T23:30:05.700Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.595326194Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134682023-09-22T23:30:05.703Z{"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.595376804Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134692023-09-22T23:30:05.707Z{"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.595445302Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134702023-09-22T23:30:05.710Z{"msg":"Next flush: 155","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.59560308Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134712023-09-22T23:30:05.714Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.595677775Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134722023-09-22T23:30:05.718Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.595705529Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134732023-09-22T23:30:05.721Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.595724052Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134742023-09-22T23:30:05.724Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.595763292Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134752023-09-22T23:30:05.729Z{"msg":"e71f97dc-3db5-4980-9004-9e02f5337158 is now active with session: b3540714-57b4-4031-acbd-3d74af480751","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.595822768Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134762023-09-22T23:30:05.732Z{"msg":"e71f97dc-3db5-4980-9004-9e02f5337158 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.595869913Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134772023-09-22T23:30:05.735Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.595921142Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134782023-09-22T23:30:05.739Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.59597141Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134792023-09-22T23:30:05.742Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.596030964Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134802023-09-22T23:30:05.747Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.596055768Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134812023-09-22T23:30:05.750Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.596073708Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134822023-09-22T23:30:05.754Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.596115671Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134832023-09-22T23:30:05.758Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
134842023-09-22T23:30:05.763Z2023-09-22T23:30:05.596181727Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134852023-09-22T23:30:05.766Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.596287459Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134862023-09-22T23:30:05.770Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.596331683Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134872023-09-22T23:30:05.773Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.59637108Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134882023-09-22T23:30:05.776Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.596409512Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1582}
134892023-09-22T23:30:05.781ZLoading write count information from file "/tmp/test_repair_verify.data"
134902023-09-22T23:30:05.784ZRun Repair workload
134912023-09-22T23:30:05.787Z01/30 Write block 211 len 32768 data: 15 11 11 9 9 8 8 7
134922023-09-22T23:30:05.792Z02/30 Flush
134932023-09-22T23:30:05.795Z03/30 Write block 240 len 24576 data: 10 11 8 7 5 5
134942023-09-22T23:30:05.798Z04/30 Write block 405 len 32768 data: 8 7 7 6 8 6 9 6
134952023-09-22T23:30:05.803Z05/30 Read block 193 len 20480
134962023-09-22T23:30:05.818Z06/30 Read block 57 len 32768
134972023-09-22T23:30:05.821Z07/30 Read block 246 len 20480
134982023-09-22T23:30:05.825Z08/30 Write block 88 len 28672 data: 8 8 7 9 10 9 9
134992023-09-22T23:30:05.828Z09/30 Write block 321 len 16384 data: 9 6 8 7
135002023-09-22T23:30:05.832Z10/30 Write block 354 len 28672 data: 5 6 5 4 6 8 9
135012023-09-22T23:30:05.835Z11/30 Read block 496 len 28672
135022023-09-22T23:30:05.838Z12/30 Read block 542 len 4096
135032023-09-22T23:30:05.841Z13/30 Flush
135042023-09-22T23:30:05.844Z14/30 Write block 469 len 4096 data: 11
135052023-09-22T23:30:05.847Z15/30 Write block 272 len 40960 data: 15 13 13 14 17 14 17 17 16 14
135062023-09-22T23:30:05.851Z16/30 Write block 535 len 24576 data: 9 9 11 12 12 7
135072023-09-22T23:30:05.854Z17/30 Read block 207 len 4096
135082023-09-22T23:30:05.857Z18/30 Write block 292 len 40960 data: 11 11 9 10 9 6 7 8 6 8
135092023-09-22T23:30:05.860Z19/30 Write block 518 len 16384 data: 13 11 10 8
135102023-09-22T23:30:05.863Z20/30 Read block 194 len 4096
135112023-09-22T23:30:05.866Z21/30 Flush
135122023-09-22T23:30:05.870Z22/30 Write block 550 len 36864 data: 12 12 12 11 6 6 7 6 7
135132023-09-22T23:30:05.873Z23/30 Read block 556 len 24576
135142023-09-22T23:30:05.876Z24/30 Read block 255 len 36864
135152023-09-22T23:30:05.879Z25/30 Write block 245 len 8192 data: 6 3
135162023-09-22T23:30:05.882Z26/30 Read block 20 len 32768
135172023-09-22T23:30:05.885Z27/30 Read block 15 len 16384
135182023-09-22T23:30:05.888Z28/30 Write block 234 len 36864 data: 5 6 7 7 8 10 11 12 9
135192023-09-22T23:30:05.891Z29/30 Read block 253 len 16384
135202023-09-22T23:30:05.895Z30/30 Read block 219 len 8192
135212023-09-22T23:30:05.898Z----------------------------------------------------------------
135222023-09-22T23:30:05.901Z Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:28
135232023-09-22T23:30:05.904ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
135242023-09-22T23:30:05.908Z 3 Acked 1002 Write 6 Done Done Done false
135252023-09-22T23:30:05.912Z 4 Acked 1003 Write 8 Done Done Done false
135262023-09-22T23:30:05.915Z 5 Acked 1004 Read 5 Done Done Done false
135272023-09-22T23:30:05.918Z 6 Acked 1005 Read 8 Done Done Done false
135282023-09-22T23:30:05.921Z 7 Acked 1006 Read 5 Done Done Sent false
135292023-09-22T23:30:05.925Z 8 Acked 1007 Write 7 Done Done Sent false
135302023-09-22T23:30:05.940Z 9 Acked 1008 Write 4 Done Done Sent false
135312023-09-22T23:30:05.943Z 10 Acked 1009 Write 7 Done Done Sent false
135322023-09-22T23:30:05.947Z 11 Acked 1010 Read 7 Done Done Sent false
135332023-09-22T23:30:05.950Z 12 Acked 1011 Read 1 Done Done Sent false
135342023-09-22T23:30:05.954Z 13 Acked 1012 Flush 0 Done Done Sent false
135352023-09-22T23:30:05.957Z 14 Acked 1013 Write 1 Done Done Sent false
135362023-09-22T23:30:05.961Z 15 Acked 1014 Write 10 Done Done Sent false
135372023-09-22T23:30:05.965Z 16 Acked 1015 Write 6 Done Done Sent false
135382023-09-22T23:30:05.969Z 17 Acked 1016 Read 1 Done Done Sent false
135392023-09-22T23:30:05.972Z 18 Acked 1017 Write 10 Done Done Sent false
135402023-09-22T23:30:05.976Z 19 Acked 1018 Write 4 Done Done Sent false
135412023-09-22T23:30:05.979Z 20 Acked 1019 Read 1 Done Done Sent false
135422023-09-22T23:30:05.982Z 21 Acked 1020 Flush 0 Done Done Sent false
135432023-09-22T23:30:05.985Z 22 Acked 1021 Write 9 Done Done Sent false
135442023-09-22T23:30:05.989Z 23 Acked 1022 Read 6 Done Done Sent false
135452023-09-22T23:30:05.992Z 24 Acked 1023 Read 9 Done Done Sent false
135462023-09-22T23:30:05.995Z 25 Acked 1024 Write 2 Done Done Sent false
135472023-09-22T23:30:05.998Z 26 Acked 1025 Read 8 Done Done Sent false
135482023-09-22T23:30:06.002Z 27 Acked 1026 Read 4 Done Done Sent false
135492023-09-22T23:30:06.006Z 28 Acked 1027 Write 9 Done Done Sent false
135502023-09-22T23:30:06.009Z 29 Acked 1028 Read 4 Done Done Sent false
135512023-09-22T23:30:06.012Z 30 Acked 1029 Read 2 Done Done Sent false
135522023-09-22T23:30:06.016Z STATES DS:0 DS:1 DS:2 TOTAL
135532023-09-22T23:30:06.020Z New 0 0 0 0
135542023-09-22T23:30:06.023Z Sent 0 0 24 24
135552023-09-22T23:30:06.028Z Done 28 28 4 60
135562023-09-22T23:30:06.031Z Skipped 0 0 0 0
135572023-09-22T23:30:06.035Z Error 0 0 0 0
135582023-09-22T23:30:06.038ZLast Flush: 1020 1020 1001
135592023-09-22T23:30:06.041ZDownstairs last five completed: 1001 1000
135602023-09-22T23:30:06.047ZUpstairs last five completed: 30 29 28 27 26 25
135612023-09-22T23:30:06.050ZError: Difference in extent metadata found!
135622023-09-22T23:30:06.053ZWrote out file "/tmp/test_repair_verify.data"
135632023-09-22T23:30:06.056Z
135642023-09-22T23:30:06.061ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
135652023-09-22T23:30:06.068Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.745170865Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1583}
135662023-09-22T23:30:06.478Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.745538673Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1583}
135672023-09-22T23:30:06.481Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.745562098Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1583}
135682023-09-22T23:30:06.485Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.770133812Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1583}
135692023-09-22T23:30:06.487Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.770166003Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1583}
135702023-09-22T23:30:06.491Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.77018486Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1583}
135712023-09-22T23:30:06.494Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.785719924Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1583}
135722023-09-22T23:30:06.498Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.785751129Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1583}
135732023-09-22T23:30:06.501Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:05.785770209Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1583}
135742023-09-22T23:30:06.504ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
135752023-09-22T23:30:06.508Z 0 000-019 100 100 100 153 153 153 F F F
135762023-09-22T23:30:06.511Z 1 020-039 101 101 101 154 154 154 F F F
135772023-09-22T23:30:06.514Z 2 040-059 93 93 93 139 139 139 F F F
135782023-09-22T23:30:06.517Z 3 060-079 100 100 100 153 153 153 F F F
135792023-09-22T23:30:06.520Z 4 080-099 102 102 100 156 156 153 F F F <---
135802023-09-22T23:30:06.523Z 5 100-119 98 98 98 150 150 150 F F F
135812023-09-22T23:30:06.525Z 6 120-139 100 100 100 153 153 153 F F F
135822023-09-22T23:30:06.529Z 7 140-159 100 100 100 153 153 153 F F F
135832023-09-22T23:30:06.554Z 8 160-179 100 100 100 153 153 153 F F F
135842023-09-22T23:30:06.557Z 9 180-199 100 100 100 153 153 153 F F F
135852023-09-22T23:30:06.560Z 10 200-219 102 102 102 155 155 155 F F F
135862023-09-22T23:30:06.563Z 11 220-239 100 100 100 153 153 153 T T F <---
135872023-09-22T23:30:06.566Z 12 240-259 102 102 93 156 156 139 T T T <---
135882023-09-22T23:30:06.569Z 13 260-279 102 102 101 157 157 154 F F F <---
135892023-09-22T23:30:06.572Z 14 280-299 102 102 101 157 157 154 F F F <---
135902023-09-22T23:30:06.575Z 15 300-319 102 102 100 157 157 153 F F F <---
135912023-09-22T23:30:06.583Z 16 320-339 102 102 99 156 156 152 F F F <---
135922023-09-22T23:30:06.586Z 17 340-359 102 102 92 156 156 138 F F F <---
135932023-09-22T23:30:06.589Z 18 360-379 102 102 90 156 156 137 F F F <---
135942023-09-22T23:30:06.592Z 19 380-399 101 101 101 154 154 154 F F F
135952023-09-22T23:30:06.595Z 20 400-419 102 102 101 156 156 154 F F T <---
135962023-09-22T23:30:06.598Z 21 420-439 98 98 98 148 148 148 F F F
135972023-09-22T23:30:06.601Z 22 440-459 98 98 98 149 149 149 F F F
135982023-09-22T23:30:06.604Z 23 460-479 102 102 98 157 157 148 F F F <---
135992023-09-22T23:30:06.607Z 24 480-499 99 99 99 152 152 152 F F F
136002023-09-22T23:30:06.609Z 25 500-519 102 102 99 157 157 152 F F F <---
136012023-09-22T23:30:06.612Z 26 520-539 102 102 97 157 157 147 F F F <---
136022023-09-22T23:30:06.615Z 27 540-559 102 102 100 157 157 153 T T F <---
136032023-09-22T23:30:06.618Z 28 560-579 100 100 100 153 153 153 F F F
136042023-09-22T23:30:06.621Z 29 580-599 94 94 94 143 143 143 F F F
136052023-09-22T23:30:06.624ZMax gen: 102, Max flush: 157
136062023-09-22T23:30:06.627ZA Difference in extent metadata is expected here
136072023-09-22T23:30:06.630ZOn loop 51
136082023-09-22T23:30:06.633Z
136092023-09-22T23:30:06.636ZVerifying data now
136102023-09-22T23:30:12.323ZNo change after verify
136112023-09-22T23:30:12.326ZLoop: 51 Downstairs dump after verify (and repair):
136122023-09-22T23:30:12.341Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.341288013Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1588}
136132023-09-22T23:30:12.345Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.341723395Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1588}
136142023-09-22T23:30:12.347Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.341748098Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1588}
136152023-09-22T23:30:12.360Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.360580078Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1588}
136162023-09-22T23:30:12.363Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.360632901Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1588}
136172023-09-22T23:30:12.366Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.360653623Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1588}
136182023-09-22T23:30:12.371Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.371700775Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1588}
136192023-09-22T23:30:12.375Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.371752312Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1588}
136202023-09-22T23:30:12.377Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.37177294Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1588}
136212023-09-22T23:30:12.382ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
136222023-09-22T23:30:12.385Z 0 000-019 100 100 100 153 153 153 F F F
136232023-09-22T23:30:12.388Z 1 020-039 101 101 101 154 154 154 F F F
136242023-09-22T23:30:12.391Z 2 040-059 93 93 93 139 139 139 F F F
136252023-09-22T23:30:12.395Z 3 060-079 100 100 100 153 153 153 F F F
136262023-09-22T23:30:12.398Z 4 080-099 102 102 102 156 156 156 F F F
136272023-09-22T23:30:12.401Z 5 100-119 98 98 98 150 150 150 F F F
136282023-09-22T23:30:12.404Z 6 120-139 100 100 100 153 153 153 F F F
136292023-09-22T23:30:12.407Z 7 140-159 100 100 100 153 153 153 F F F
136302023-09-22T23:30:12.410Z 8 160-179 100 100 100 153 153 153 F F F
136312023-09-22T23:30:12.413Z 9 180-199 100 100 100 153 153 153 F F F
136322023-09-22T23:30:12.415Z 10 200-219 102 102 102 155 155 155 F F F
136332023-09-22T23:30:12.418Z 11 220-239 103 103 103 158 158 158 F F F
136342023-09-22T23:30:12.421Z 12 240-259 103 103 103 158 158 158 F F F
136352023-09-22T23:30:12.425Z 13 260-279 102 102 102 157 157 157 F F F
136362023-09-22T23:30:12.428Z 14 280-299 102 102 102 157 157 157 F F F
136372023-09-22T23:30:12.431Z 15 300-319 102 102 102 157 157 157 F F F
136382023-09-22T23:30:12.434Z 16 320-339 102 102 102 156 156 156 F F F
136392023-09-22T23:30:12.437Z 17 340-359 102 102 102 156 156 156 F F F
136402023-09-22T23:30:12.440Z 18 360-379 102 102 102 156 156 156 F F F
136412023-09-22T23:30:12.443Z 19 380-399 101 101 101 154 154 154 F F F
136422023-09-22T23:30:12.446Z 20 400-419 102 102 102 156 156 156 F F F
136432023-09-22T23:30:12.449Z 21 420-439 98 98 98 148 148 148 F F F
136442023-09-22T23:30:12.453Z 22 440-459 98 98 98 149 149 149 F F F
136452023-09-22T23:30:12.456Z 23 460-479 102 102 102 157 157 157 F F F
136462023-09-22T23:30:12.460Z 24 480-499 99 99 99 152 152 152 F F F
136472023-09-22T23:30:12.463Z 25 500-519 102 102 102 157 157 157 F F F
136482023-09-22T23:30:12.466Z 26 520-539 102 102 102 157 157 157 F F F
136492023-09-22T23:30:12.470Z 27 540-559 103 103 103 158 158 158 F F F
136502023-09-22T23:30:12.473Z 28 560-579 100 100 100 153 153 153 F F F
136512023-09-22T23:30:12.476Z 29 580-599 94 94 94 143 143 143 F F F
136522023-09-22T23:30:12.478ZMax gen: 103, Max flush: 158
136532023-09-22T23:30:12.481Z
136542023-09-22T23:30:12.484ZBegin loop 52
136552023-09-22T23:30:12.487ZDownstairs to restart: 1
136562023-09-22T23:30:12.490Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.425025466Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136572023-09-22T23:30:12.493Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.425434094Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136582023-09-22T23:30:12.496Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.425486574Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136592023-09-22T23:30:12.499Z{"msg":"Crucible stats registered with UUID: 29b74a10-1827-470f-b073-e81c6d371eef","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.425541651Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136602023-09-22T23:30:12.502Z{"msg":"Crucible 29b74a10-1827-470f-b073-e81c6d371eef has session id: 9e3efeaa-f70c-4bb7-80fe-d33fcebc5f49","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.425592139Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136612023-09-22T23:30:12.504ZCrucible runtime is spawned
136622023-09-22T23:30:12.507Z{"msg":"up_listen starts","v":The guest has requested activation with gen:104
136632023-09-22T23:30:12.510Z0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.425873269Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590,"task":"up_listen"}
136642023-09-22T23:30:12.513Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.425932297Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136652023-09-22T23:30:12.516Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.425981866Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136662023-09-22T23:30:12.519Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.426003915Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590,"looper":"0"}
136672023-09-22T23:30:12.522Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.426389344Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590,"looper":"2"}
136682023-09-22T23:30:12.525Z{"msg":"[0] 29b74a10-1827-470f-b073-e81c6d371eef looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.426440551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590,"looper":"0"}
136692023-09-22T23:30:12.531Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.426462093Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590,"looper":"1"}
136702023-09-22T23:30:12.535Z{"msg":"[2] 29b74a10-1827-470f-b073-e81c6d371eef looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.426481568Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590,"looper":"2"}
136712023-09-22T23:30:12.538Z{"msg":"29b74a10-1827-470f-b073-e81c6d371eef active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.426818736Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136722023-09-22T23:30:12.542Z{"msg":"Set desired generation to :104","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.426872345Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136732023-09-22T23:30:12.545Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.427010787Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136742023-09-22T23:30:12.548Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.427367218Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136752023-09-22T23:30:12.551Z{"msg":"[0] 29b74a10-1827-470f-b073-e81c6d371eef (9ed41987-32af-4bdd-a668-f3f8663da6b9) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.427757857Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136762023-09-22T23:30:12.554Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.42781498Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136772023-09-22T23:30:12.557Z{"msg":"[0] client is_active_req TRUE, promote! session 9ed41987-32af-4bdd-a668-f3f8663da6b9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.427885378Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136782023-09-22T23:30:12.560Z{"msg":"[2] 29b74a10-1827-470f-b073-e81c6d371eef (9ed41987-32af-4bdd-a668-f3f8663da6b9) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.427969498Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136792023-09-22T23:30:12.564Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.428036699Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136802023-09-22T23:30:12.567Z{"msg":"[2] client is_active_req TRUE, promote! session 9ed41987-32af-4bdd-a668-f3f8663da6b9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.428079915Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136812023-09-22T23:30:12.570Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.42845022Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136822023-09-22T23:30:12.573Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.428583889Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136832023-09-22T23:30:12.576Z{"msg":"29b74a10-1827-470f-b073-e81c6d371eef WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.428661669Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136842023-09-22T23:30:12.579Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.428736099Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136852023-09-22T23:30:12.582Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.428817354Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136862023-09-22T23:30:12.585Z{"msg":"29b74a10-1827-470f-b073-e81c6d371eef WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.428890128Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136872023-09-22T23:30:12.588Z{"msg":"[2] 29b74a10-1827-470f-b073-e81c6d371eef (9ed41987-32af-4bdd-a668-f3f8663da6b9) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.431004147Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136882023-09-22T23:30:12.591Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.431091688Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136892023-09-22T23:30:12.594Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:30:12.43116654Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136902023-09-22T23:30:12.597Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.431233704Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136912023-09-22T23:30:12.601Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.431270435Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136922023-09-22T23:30:12.604Z{"msg":"29b74a10-1827-470f-b073-e81c6d371eef WaitActive New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.43133384Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136932023-09-22T23:30:12.607Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.431403047Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136942023-09-22T23:30:12.610Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.431441973Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136952023-09-22T23:30:12.613Z{"msg":"[0] 29b74a10-1827-470f-b073-e81c6d371eef (9ed41987-32af-4bdd-a668-f3f8663da6b9) WaitActive New WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.431514489Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136962023-09-22T23:30:12.616Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.431555946Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136972023-09-22T23:30:12.619Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:30:12.431624642Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136982023-09-22T23:30:12.622Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.43172005Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
136992023-09-22T23:30:12.625Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.431788023Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137002023-09-22T23:30:12.628Z{"msg":"29b74a10-1827-470f-b073-e81c6d371eef WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.431856206Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137012023-09-22T23:30:12.631Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.431931918Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137022023-09-22T23:30:12.634Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:12.432004196Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137032023-09-22T23:30:13.427Z{"msg":"[1] 29b74a10-1827-470f-b073-e81c6d371eef looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.427635036Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590,"looper":"1"}
137042023-09-22T23:30:13.431Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.427728317Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137052023-09-22T23:30:13.434Z{"msg":"[1] 29b74a10-1827-470f-b073-e81c6d371eef (9ed41987-32af-4bdd-a668-f3f8663da6b9) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.428858312Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137062023-09-22T23:30:13.437Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.428915045Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137072023-09-22T23:30:13.440Z{"msg":"[1] client is_active_req TRUE, promote! session 9ed41987-32af-4bdd-a668-f3f8663da6b9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.428959328Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137082023-09-22T23:30:13.443Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.429515928Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137092023-09-22T23:30:13.447Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.429572104Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137102023-09-22T23:30:13.450Z{"msg":"29b74a10-1827-470f-b073-e81c6d371eef WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.429615584Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137112023-09-22T23:30:13.453Z{"msg":"[1] 29b74a10-1827-470f-b073-e81c6d371eef (9ed41987-32af-4bdd-a668-f3f8663da6b9) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.431992682Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137122023-09-22T23:30:13.456Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.432024799Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137132023-09-22T23:30:13.459Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:30:13.432081353Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137142023-09-22T23:30:13.462Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.432102238Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137152023-09-22T23:30:13.465Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.432140992Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137162023-09-22T23:30:13.468Z{"msg":"29b74a10-1827-470f-b073-e81c6d371eef WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.432163457Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137172023-09-22T23:30:13.472Z{"msg":"[0]R flush_numbers[0..12]: [153, 154, 139, 153, 156, 150, 153, 153, 153, 153, 155, 158]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.43220285Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137182023-09-22T23:30:13.475Z{"msg":"[0]R generation[0..12]: [100, 101, 93, 100, 102, 98, 100, 100, 100, 100, 102, 103]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.432224889Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137192023-09-22T23:30:13.478Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.432263908Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137202023-09-22T23:30:13.481Z{"msg":"[1]R flush_numbers[0..12]: [153, 154, 139, 153, 156, 150, 153, 153, 153, 153, 155, 158]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.432303813Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137212023-09-22T23:30:13.484Z{"msg":"[1]R generation[0..12]: [100, 101, 93, 100, 102, 98, 100, 100, 100, 100, 102, 103]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.432343796Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137222023-09-22T23:30:13.487Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.43236687Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137232023-09-22T23:30:13.491Z{"msg":"[2]R flush_numbers[0..12]: [153, 154, 139, 153, 156, 150, 153, 153, 153, 153, 155, 158]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.432419925Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137242023-09-22T23:30:13.494Z{"msg":"[2]R generation[0..12]: [100, 101, 93, 100, 102, 98, 100, 100, 100, 100, 102, 103]","v":0,"The guest has finished waiting for activation with:104
137252023-09-22T23:30:13.497ZWait for a query_work_queue command to finish before sending IO
137262023-09-22T23:30:13.501Zname":"crucible","level":30,"time":"2023-09-22T23:30:13.432640011Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137272023-09-22T23:30:13.504Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.43271341Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137282023-09-22T23:30:13.508Z{"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.432761653Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137292023-09-22T23:30:13.510Z{"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.4328022Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137302023-09-22T23:30:13.514Z{"msg":"Next flush: 159","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.432864221Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137312023-09-22T23:30:13.517Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.432930759Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137322023-09-22T23:30:13.520Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.432994234Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137332023-09-22T23:30:13.523Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.433038665Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137342023-09-22T23:30:13.526Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.433093762Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137352023-09-22T23:30:13.529Z{"msg":"29b74a10-1827-470f-b073-e81c6d371eef is now active with session: 9ed41987-32af-4bdd-a668-f3f8663da6b9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.433163355Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137362023-09-22T23:30:13.532Z{"msg":"29b74a10-1827-470f-b073-e81c6d371eef Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.433226831Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137372023-09-22T23:30:13.536Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.433271829Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137382023-09-22T23:30:13.539Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.433332042Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137392023-09-22T23:30:13.542Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.433398689Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137402023-09-22T23:30:13.545Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.43344955Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137412023-09-22T23:30:13.549Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.433504245Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137422023-09-22T23:30:13.552Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.43357348Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137432023-09-22T23:30:13.555Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.433624019Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137442023-09-22T23:30:13.558Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
137452023-09-22T23:30:13.561Z,"time":"2023-09-22T23:30:13.433680354Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137462023-09-22T23:30:13.565Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.43380789Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137472023-09-22T23:30:13.568Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.433878229Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137482023-09-22T23:30:13.572Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.43394083Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1590}
137492023-09-22T23:30:13.575ZLoading write count information from file "/tmp/test_repair_verify.data"
137502023-09-22T23:30:13.579ZRun Repair workload
137512023-09-22T23:30:13.583Z01/30 Write block 157 len 28672 data: 7 10 12 11 11 11 9
137522023-09-22T23:30:13.586Z02/30 Read block 190 len 36864
137532023-09-22T23:30:13.589Z03/30 Read block 379 len 24576
137542023-09-22T23:30:13.592Z04/30 Flush
137552023-09-22T23:30:13.595Z05/30 Write block 511 len 16384 data: 7 10 12 14
137562023-09-22T23:30:13.599Z06/30 Read block 495 len 36864
137572023-09-22T23:30:13.602Z07/30 Write block 290 len 12288 data: 9 10 12
137582023-09-22T23:30:13.605Z08/30 Read block 265 len 12288
137592023-09-22T23:30:13.608Z09/30 Read block 16 len 20480
137602023-09-22T23:30:13.612Z10/30 Write block 302 len 8192 data: 9 10
137612023-09-22T23:30:13.615Z11/30 Write block 109 len 28672 data: 13 13 10 9 9 7 6
137622023-09-22T23:30:13.618Z12/30 Write block 329 len 12288 data: 7 6 8
137632023-09-22T23:30:13.621Z13/30 Read block 24 len 8192
137642023-09-22T23:30:13.624Z14/30 Write block 224 len 20480 data: 7 8 8 7 7
137652023-09-22T23:30:13.627Z15/30 Read block 245 len 4096
137662023-09-22T23:30:13.631Z16/30 Read block 239 len 24576
137672023-09-22T23:30:13.634Z17/30 Read block 90 len 4096
137682023-09-22T23:30:13.638Z18/30 Read block 348 len 4096
137692023-09-22T23:30:13.641Z19/30 Read block 12 len 4096
137702023-09-22T23:30:13.644Z20/30 Write block 35 len 12288 data: 8 11 12
137712023-09-22T23:30:13.647Z21/30 Read block 331 len 28672
137722023-09-22T23:30:13.651Z22/30 Write block 372 len 36864 data: 6 6 7 7 7 9 10 9 9
137732023-09-22T23:30:13.654Z23/30 Read block 73 len 20480
137742023-09-22T23:30:13.657Z24/30 Write block 428 len 4096 data: 9
137752023-09-22T23:30:13.660Z25/30 Write block 453 len 12288 data: 9 8 7
137762023-09-22T23:30:13.663Z26/30 Write block 457 len 16384 data: 7 7 11 10
137772023-09-22T23:30:13.666Z27/30 Write block 437 len 40960 data: 9 8 6 6 9 10 12 12 11 8
137782023-09-22T23:30:13.669Z28/30 Write block 214 len 16384 data: 10 10 9 9
137792023-09-22T23:30:13.672Z29/30 Read block 79 len 36864
137802023-09-22T23:30:13.675Z30/30 Write block 169 len 40960 data: 4 8 9 11 11 11 12 12 11 11
137812023-09-22T23:30:13.678Z----------------------------------------------------------------
137822023-09-22T23:30:13.682Z Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:26
137832023-09-22T23:30:13.686ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
137842023-09-22T23:30:13.689Z 5 Acked 1004 Write 4 Done Done Done false
137852023-09-22T23:30:13.692Z 6 Acked 1005 Read 9 Done Sent Done false
137862023-09-22T23:30:13.721Z 7 Acked 1006 Write 3 Done Sent Done false
137872023-09-22T23:30:13.725Z 8 Acked 1007 Read 3 Done Sent Done false
137882023-09-22T23:30:13.728Z 9 Acked 1008 Read 5 Done Sent Done false
137892023-09-22T23:30:13.732Z 10 Acked 1009 Write 2 Done Sent Done false
137902023-09-22T23:30:13.734Z 11 Acked 1010 Write 7 Done Sent Done false
137912023-09-22T23:30:13.737Z 12 Acked 1011 Write 3 Done Sent Done false
137922023-09-22T23:30:13.741Z 13 Acked 1012 Read 2 Done Sent Done false
137932023-09-22T23:30:13.743Z 14 Acked 1013 Write 5 Done Sent Done false
137942023-09-22T23:30:13.746Z 15 Acked 1014 Read 1 Done Sent Done false
137952023-09-22T23:30:13.749Z 16 Acked 1015 Read 6 Done Sent Done false
137962023-09-22T23:30:13.753Z 17 Acked 1016 Read 1 Done Sent Done false
137972023-09-22T23:30:13.756Z 18 Acked 1017 Read 1 Done Sent Done false
137982023-09-22T23:30:13.761Z 19 Acked 1018 Read 1 Done Sent Done false
137992023-09-22T23:30:13.765Z 20 Acked 1019 Write 3 Done Sent Done false
138002023-09-22T23:30:13.767Z 21 Acked 1020 Read 7 Done Sent Done false
138012023-09-22T23:30:13.770Z 22 Acked 1021 Write 9 Done Sent Done false
138022023-09-22T23:30:13.773Z 23 Acked 1022 Read 5 Done Sent Done false
138032023-09-22T23:30:13.776Z 24 Acked 1023 Write 1 Done Sent Done false
138042023-09-22T23:30:13.780Z 25 Acked 1024 Write 3 Done Sent Done false
138052023-09-22T23:30:13.784Z 26 Acked 1025 Write 4 Done Sent Done false
138062023-09-22T23:30:13.787Z 27 Acked 1026 Write 10 Done Sent Done false
138072023-09-22T23:30:13.790Z 28 Acked 1027 Write 4 Done Sent Done false
138082023-09-22T23:30:13.793Z 29 Acked 1028 Read 9 Done Sent Done false
138092023-09-22T23:30:13.797Z 30 Acked 1029 Write 10 Sent Sent Sent false
138102023-09-22T23:30:13.800Z STATES DS:0 DS:1 DS:2 TOTAL
138112023-09-22T23:30:13.804Z New 0 0 0 0
138122023-09-22T23:30:13.807Z Sent 1 25 1 27
138132023-09-22T23:30:13.810Z Done 25 1 25 51
138142023-09-22T23:30:13.813Z Skipped 0 0 0 0
138152023-09-22T23:30:13.816Z Error 0 0 0 0
138162023-09-22T23:30:13.820ZLast Flush: 1003 1003 1003
138172023-09-22T23:30:13.823ZDownstairs last five completed: 1003 1002 1001 1000
138182023-09-22T23:30:13.826ZUpstairs last five completed: 30 29 28 27 26 25
138192023-09-22T23:30:13.830ZWrote out file "/tmp/test_repair_verify.data"
138202023-09-22T23:30:13.834Z
138212023-09-22T23:30:13.837ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
138222023-09-22T23:30:13.841Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.570365153Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1591}
138232023-09-22T23:30:13.844Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.570768352Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1591}
138242023-09-22T23:30:13.848Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.570791201Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1591}
138252023-09-22T23:30:13.851Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.612302944Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1591}
138262023-09-22T23:30:13.854Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.612359696Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1591}
138272023-09-22T23:30:13.857Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.612380964Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1591}
138282023-09-22T23:30:13.860Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.637680143Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1591}
138292023-09-22T23:30:13.863Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.637733669Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1591}
138302023-09-22T23:30:13.866Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:13.637772352Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1591}
138312023-09-22T23:30:13.872ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
138322023-09-22T23:30:13.877Z 0 000-019 100 100 100 153 153 153 F F F
138332023-09-22T23:30:13.880Z 1 020-039 101 101 101 154 154 154 T F T <---
138342023-09-22T23:30:13.883Z 2 040-059 93 93 93 139 139 139 F F F
138352023-09-22T23:30:13.886Z 3 060-079 100 100 100 153 153 153 F F F
138362023-09-22T23:30:13.889Z 4 080-099 102 102 102 156 156 156 F F F
138372023-09-22T23:30:13.892Z 5 100-119 98 98 98 150 150 150 T F T <---
138382023-09-22T23:30:13.896Z 6 120-139 100 100 100 153 153 153 F F F
138392023-09-22T23:30:13.899Z 7 140-159 104 104 104 159 159 159 F F F
138402023-09-22T23:30:13.902Z 8 160-179 104 104 104 159 159 159 T F T <---
138412023-09-22T23:30:13.919Z 9 180-199 100 100 100 153 153 153 F F F
138422023-09-22T23:30:13.922Z 10 200-219 102 102 102 155 155 155 T F T <---
138432023-09-22T23:30:13.927Z 11 220-239 103 103 103 158 158 158 T F T <---
138442023-09-22T23:30:13.931ZError: Difference in extent metadata found!
138452023-09-22T23:30:13.935Z 12 240-259 103 103 103 158 158 158 F F F
138462023-09-22T23:30:13.938Z 13 260-279 102 102 102 157 157 157 F F F
138472023-09-22T23:30:13.941Z 14 280-299 102 102 102 157 157 157 T F T <---
138482023-09-22T23:30:13.944Z 15 300-319 102 102 102 157 157 157 T F T <---
138492023-09-22T23:30:13.948Z 16 320-339 102 102 102 156 156 156 T F T <---
138502023-09-22T23:30:13.950Z 17 340-359 102 102 102 156 156 156 F F F
138512023-09-22T23:30:13.954Z 18 360-379 102 102 102 156 156 156 T F T <---
138522023-09-22T23:30:13.957Z 19 380-399 101 101 101 154 154 154 T F T <---
138532023-09-22T23:30:13.961Z 20 400-419 102 102 102 156 156 156 F F F
138542023-09-22T23:30:13.964Z 21 420-439 98 98 98 148 148 148 T F T <---
138552023-09-22T23:30:13.968Z 22 440-459 98 98 98 149 149 149 T F T <---
138562023-09-22T23:30:13.971Z 23 460-479 102 102 102 157 157 157 T F T <---
138572023-09-22T23:30:13.975Z 24 480-499 99 99 99 152 152 152 F F F
138582023-09-22T23:30:13.979Z 25 500-519 102 102 102 157 157 157 T T T
138592023-09-22T23:30:13.982Z 26 520-539 102 102 102 157 157 157 F F F
138602023-09-22T23:30:13.985Z 27 540-559 103 103 103 158 158 158 F F F
138612023-09-22T23:30:13.988Z 28 560-579 100 100 100 153 153 153 F F F
138622023-09-22T23:30:13.991Z 29 580-599 94 94 94 143 143 143 F F F
138632023-09-22T23:30:13.994ZMax gen: 104, Max flush: 159
138642023-09-22T23:30:13.998ZA Difference in extent metadata is expected here
138652023-09-22T23:30:14.002ZOn loop 52
138662023-09-22T23:30:14.005Z
138672023-09-22T23:30:14.008ZVerifying data now
138682023-09-22T23:30:20.200ZNo change after verify
138692023-09-22T23:30:20.203ZLoop: 52 Downstairs dump after verify (and repair):
138702023-09-22T23:30:20.219Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.218876566Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1596}
138712023-09-22T23:30:20.222Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.219304746Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1596}
138722023-09-22T23:30:20.227Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.219330128Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1596}
138732023-09-22T23:30:20.238Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.238289437Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1596}
138742023-09-22T23:30:20.241Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.238343754Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1596}
138752023-09-22T23:30:20.244Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.238364392Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1596}
138762023-09-22T23:30:20.249Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.249572169Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1596}
138772023-09-22T23:30:20.252Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.249626006Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1596}
138782023-09-22T23:30:20.254Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.249647575Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1596}
138792023-09-22T23:30:20.260ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
138802023-09-22T23:30:20.263Z 0 000-019 100 100 100 153 153 153 F F F
138812023-09-22T23:30:20.267Z 1 020-039 105 105 105 160 160 160 F F F
138822023-09-22T23:30:20.270Z 2 040-059 93 93 93 139 139 139 F F F
138832023-09-22T23:30:20.273Z 3 060-079 100 100 100 153 153 153 F F F
138842023-09-22T23:30:20.276Z 4 080-099 102 102 102 156 156 156 F F F
138852023-09-22T23:30:20.279Z 5 100-119 105 105 105 160 160 160 F F F
138862023-09-22T23:30:20.282Z 6 120-139 100 100 100 153 153 153 F F F
138872023-09-22T23:30:20.285Z 7 140-159 104 104 104 159 159 159 F F F
138882023-09-22T23:30:20.288Z 8 160-179 105 105 105 160 160 160 F F F
138892023-09-22T23:30:20.291Z 9 180-199 100 100 100 153 153 153 F F F
138902023-09-22T23:30:20.301Z 10 200-219 105 105 105 160 160 160 F F F
138912023-09-22T23:30:20.304Z 11 220-239 105 105 105 160 160 160 F F F
138922023-09-22T23:30:20.307Z 12 240-259 103 103 103 158 158 158 F F F
138932023-09-22T23:30:20.312Z 13 260-279 102 102 102 157 157 157 F F F
138942023-09-22T23:30:20.315Z 14 280-299 105 105 105 160 160 160 F F F
138952023-09-22T23:30:20.318Z 15 300-319 105 105 105 160 160 160 F F F
138962023-09-22T23:30:20.321Z 16 320-339 105 105 105 160 160 160 F F F
138972023-09-22T23:30:20.324Z 17 340-359 102 102 102 156 156 156 F F F
138982023-09-22T23:30:20.328Z 18 360-379 105 105 105 160 160 160 F F F
138992023-09-22T23:30:20.331Z 19 380-399 105 105 105 160 160 160 F F F
139002023-09-22T23:30:20.334Z 20 400-419 102 102 102 156 156 156 F F F
139012023-09-22T23:30:20.337Z 21 420-439 105 105 105 160 160 160 F F F
139022023-09-22T23:30:20.340Z 22 440-459 105 105 105 160 160 160 F F F
139032023-09-22T23:30:20.342Z 23 460-479 105 105 105 160 160 160 F F F
139042023-09-22T23:30:20.345Z 24 480-499 99 99 99 152 152 152 F F F
139052023-09-22T23:30:20.348Z 25 500-519 105 105 105 160 160 160 F F F
139062023-09-22T23:30:20.351Z 26 520-539 102 102 102 157 157 157 F F F
139072023-09-22T23:30:20.354Z 27 540-559 103 103 103 158 158 158 F F F
139082023-09-22T23:30:20.357Z 28 560-579 100 100 100 153 153 153 F F F
139092023-09-22T23:30:20.360Z 29 580-599 94 94 94 143 143 143 F F F
139102023-09-22T23:30:20.363ZMax gen: 105, Max flush: 160
139112023-09-22T23:30:20.366Z
139122023-09-22T23:30:20.369ZBegin loop 53
139132023-09-22T23:30:20.371ZDownstairs to restart: 0
139142023-09-22T23:30:20.375Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.300878029Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139152023-09-22T23:30:20.386Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.301251677Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139162023-09-22T23:30:20.389Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.301323601Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139172023-09-22T23:30:20.392Z{"msg":"Crucible stats registered with UUID: 4a639311-82de-4926-b8be-ad159a732985","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.301383855Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139182023-09-22T23:30:20.395Z{"msg":"Crucible 4a639311-82de-4926-b8be-ad159a732985 has session id: 8253c0e2-b678-471e-9265-ab8c79d81b1b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.301452145Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139192023-09-22T23:30:20.398ZCrucible runtime is spawned
139202023-09-22T23:30:20.402Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.301709004Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598,"task":"up_listen"}
139212023-09-22T23:30:20.405Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.301763588Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598The guest has requested activation with gen:106
139222023-09-22T23:30:20.408Z}
139232023-09-22T23:30:20.411Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.301906442Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139242023-09-22T23:30:20.415Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.301957553Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598,"looper":"1"}
139252023-09-22T23:30:20.418Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.302228792Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598,"looper":"2"}
139262023-09-22T23:30:20.421Z{"msg":"[1] 4a639311-82de-4926-b8be-ad159a732985 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.30233178Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598,"looper":"1"}
139272023-09-22T23:30:20.423Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.302418971Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598,"looper":"0"}
139282023-09-22T23:30:20.426Z{"msg":"[2] 4a639311-82de-4926-b8be-ad159a732985 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.302490106Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598,"looper":"2"}
139292023-09-22T23:30:20.430Z{"msg":"4a639311-82de-4926-b8be-ad159a732985 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.302839014Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139302023-09-22T23:30:20.434Z{"msg":"Set desired generation to :106","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.302898638Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139312023-09-22T23:30:20.437Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.303022455Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139322023-09-22T23:30:20.441Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.303421792Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139332023-09-22T23:30:20.443Z{"msg":"[1] 4a639311-82de-4926-b8be-ad159a732985 (99040eb9-76e6-48f2-8a74-9ee68cea7236) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.303647757Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139342023-09-22T23:30:20.446Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.303713974Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139352023-09-22T23:30:20.449Z{"msg":"[1] client is_active_req TRUE, promote! session 99040eb9-76e6-48f2-8a74-9ee68cea7236","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.303774213Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139362023-09-22T23:30:20.452Z{"msg":"[2] 4a639311-82de-4926-b8be-ad159a732985 (99040eb9-76e6-48f2-8a74-9ee68cea7236) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.303900932Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139372023-09-22T23:30:20.455Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.303956424Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139382023-09-22T23:30:20.462Z{"msg":"[2] client is_active_req TRUE, promote! session 99040eb9-76e6-48f2-8a74-9ee68cea7236","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.304019253Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139392023-09-22T23:30:20.465Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.304266699Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139402023-09-22T23:30:20.468Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.304333677Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139412023-09-22T23:30:20.471Z{"msg":"4a639311-82de-4926-b8be-ad159a732985 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.304379687Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139422023-09-22T23:30:20.475Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.304441567Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139432023-09-22T23:30:20.477Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.30451676Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139442023-09-22T23:30:20.480Z{"msg":"4a639311-82de-4926-b8be-ad159a732985 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.304562059Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139452023-09-22T23:30:20.483Z{"msg":"[2] 4a639311-82de-4926-b8be-ad159a732985 (99040eb9-76e6-48f2-8a74-9ee68cea7236) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.30672644Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139462023-09-22T23:30:20.486Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.306783492Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139472023-09-22T23:30:20.489Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:30:20.306825552Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139482023-09-22T23:30:20.492Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.306864446Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139492023-09-22T23:30:20.495Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.30688511Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139502023-09-22T23:30:20.907Z{"msg":"4a639311-82de-4926-b8be-ad159a732985 New WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.306905251Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139512023-09-22T23:30:20.910Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.306942807Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139522023-09-22T23:30:20.915Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.306981909Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139532023-09-22T23:30:20.919Z{"msg":"[1] 4a639311-82de-4926-b8be-ad159a732985 (99040eb9-76e6-48f2-8a74-9ee68cea7236) New WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.307885423Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139542023-09-22T23:30:20.921Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.307954199Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139552023-09-22T23:30:20.925Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:30:20.308000343Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139562023-09-22T23:30:20.928Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.308039905Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139572023-09-22T23:30:20.932Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.308079095Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139582023-09-22T23:30:20.936Z{"msg":"4a639311-82de-4926-b8be-ad159a732985 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.308118438Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139592023-09-22T23:30:20.939Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.30813892Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139602023-09-22T23:30:20.942Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:20.308156709Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139612023-09-22T23:30:21.305Z{"msg":"[0] 4a639311-82de-4926-b8be-ad159a732985 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.304374053Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598,"looper":"0"}
139622023-09-22T23:30:21.308Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.304457881Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139632023-09-22T23:30:21.312Z{"msg":"[0] 4a639311-82de-4926-b8be-ad159a732985 (99040eb9-76e6-48f2-8a74-9ee68cea7236) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.3054729Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139642023-09-22T23:30:21.315Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.305537914Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139652023-09-22T23:30:21.318Z{"msg":"[0] client is_active_req TRUE, promote! session 99040eb9-76e6-48f2-8a74-9ee68cea7236","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.305583159Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139662023-09-22T23:30:21.321Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.306128691Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139672023-09-22T23:30:21.326Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.30618476Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139682023-09-22T23:30:21.328Z{"msg":"4a639311-82de-4926-b8be-ad159a732985 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.306228342Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139692023-09-22T23:30:21.331Z{"msg":"[0] 4a639311-82de-4926-b8be-ad159a732985 (99040eb9-76e6-48f2-8a74-9ee68cea7236) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.30860275Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139702023-09-22T23:30:21.334Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.308657572Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139712023-09-22T23:30:21.337Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:30:21.30870363Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139722023-09-22T23:30:21.340Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.308725445Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139732023-09-22T23:30:21.343Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.308763231Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139742023-09-22T23:30:21.346Z{"msg":"4a639311-82de-4926-b8be-ad159a732985 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.308785786Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139752023-09-22T23:30:21.349Z{"msg":"[0]R flush_numbers[0..12]: [153, 160, 139, 153, 156, 160, 153, 159, 160, 153, 160, 160]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.308829387Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139762023-09-22T23:30:21.354Z{"msg":"[0]R generation[0..12]: [100, 105, 93, 100, 102, 105, 100, 104, 105, 100, 105, 105]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.308869865Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139772023-09-22T23:30:21.358Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.308910453Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139782023-09-22T23:30:21.360Z{"msg":"[1]R flush_numbers[0..12]: [153, 160, 139, 153, 156, 160, 153, 159, 160, 153, 160, 160]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.308971254Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139792023-09-22T23:30:21.363Z{"msg":"The guest has finished waiting for activation with:106
139802023-09-22T23:30:21.366ZWait for a query_work_queue command to finish before sending IO
139812023-09-22T23:30:21.369Z[1]R generation[0..12]: [100, 105, 93, 100, 102, 105, 100, 104, 105, 100, 105, 105]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.309220435Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139822023-09-22T23:30:21.372Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.309285849Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139832023-09-22T23:30:21.376Z{"msg":"[2]R flush_numbers[0..12]: [153, 160, 139, 153, 156, 160, 153, 159, 160, 153, 160, 160]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.309383493Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139842023-09-22T23:30:21.379Z{"msg":"[2]R generation[0..12]: [100, 105, 93, 100, 102, 105, 100, 104, 105, 100, 105, 105]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.309406836Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139852023-09-22T23:30:21.382Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.309456167Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139862023-09-22T23:30:21.385Z{"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.309494012Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139872023-09-22T23:30:21.389Z{"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.309534602Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139882023-09-22T23:30:21.392Z{"msg":"Next flush: 161","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.309554415Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139892023-09-22T23:30:21.394Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.309571777Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139902023-09-22T23:30:21.397Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.309610391Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139912023-09-22T23:30:21.401Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.309648297Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139922023-09-22T23:30:21.404Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.30968983Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139932023-09-22T23:30:21.407Z{"msg":"4a639311-82de-4926-b8be-ad159a732985 is now active with session: 99040eb9-76e6-48f2-8a74-9ee68cea7236","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.309729671Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139942023-09-22T23:30:21.410Z{"msg":"4a639311-82de-4926-b8be-ad159a732985 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.309775315Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139952023-09-22T23:30:21.415Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.309795747Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139962023-09-22T23:30:21.418Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.309831793Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139972023-09-22T23:30:21.421Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.309875149Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139982023-09-22T23:30:21.424Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.309913204Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
139992023-09-22T23:30:21.427Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.309956376Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
140002023-09-22T23:30:21.430Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.309994066Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
140012023-09-22T23:30:21.434Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.310031521Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
140022023-09-22T23:30:21.437Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.310078131Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
140032023-09-22T23:30:21.440Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.310115643Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
140042023-09-22T23:30:21.443Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.310161065Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
140052023-09-22T23:30:21.447Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.310181128Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1598}
140062023-09-22T23:30:21.865ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
140072023-09-22T23:30:21.871ZLoading write count information from file "/tmp/test_repair_verify.data"
140082023-09-22T23:30:21.886ZRun Repair workload
140092023-09-22T23:30:21.895Z01/30 Write block 34 len 24576 data: 7 9 12 13 12 11
140102023-09-22T23:30:21.900Z02/30 Read block 456 len 12288
140112023-09-22T23:30:21.906Z03/30 Read block 71 len 12288
140122023-09-22T23:30:21.909Z04/30 Write block 108 len 36864 data: 13 14 14 11 10 10 8 7 7
140132023-09-22T23:30:21.912Z05/30 Read block 582 len 12288
140142023-09-22T23:30:21.915Z06/30 Read block 38 len 8192
140152023-09-22T23:30:21.918Z07/30 Write block 12 len 36864 data: 8 9 7 8 7 7 8 6 5
140162023-09-22T23:30:21.921Z08/30 Read block 75 len 36864
140172023-09-22T23:30:21.924Z09/30 Read block 57 len 24576
140182023-09-22T23:30:21.927Z10/30 Flush
140192023-09-22T23:30:21.930Z11/30 Write block 0 len 28672 data: 3 4 5 6 5 6 6
140202023-09-22T23:30:21.933Z12/30 Read block 584 len 24576
140212023-09-22T23:30:21.936Z13/30 Flush
140222023-09-22T23:30:21.939Z14/30 Write block 222 len 24576 data: 7 8 8 9 9 8
140232023-09-22T23:30:21.942Z15/30 Write block 96 len 40960 data: 11 10 11 9 8 8 9 10 12 14
140242023-09-22T23:30:21.946Z16/30 Read block 71 len 4096
140252023-09-22T23:30:21.949Z17/30 Write block 478 len 12288 data: 8 9 8
140262023-09-22T23:30:21.952Z18/30 Write block 66 len 12288 data: 19 18 18
140272023-09-22T23:30:21.955Z19/30 Write block 116 len 32768 data: 8 8 7 6 7 7 9 10
140282023-09-22T23:30:21.958Z20/30 Read block 517 len 40960
140292023-09-22T23:30:21.961Z21/30 Write block 345 len 8192 data: 9 10
140302023-09-22T23:30:21.964Z22/30 Read block 287 len 40960
140312023-09-22T23:30:21.989Z23/30 Write block 422 len 36864 data: 7 7 7 4 6 8 10 10 11
140322023-09-22T23:30:21.993Z24/30 Write block 593 len 16384 data: 13 10 10 10
140332023-09-22T23:30:21.996Z25/30 Flush
140342023-09-22T23:30:22.001Z26/30 Write block 418 len 40960 data: 10 11 10 10 8 8 8 5 7 9
140352023-09-22T23:30:22.004Z27/30 Read block 156 len 32768
140362023-09-22T23:30:22.011Z28/30 Read block 345 len 24576
140372023-09-22T23:30:22.015Z29/30 Write block 232 len 4096 data: 5
140382023-09-22T23:30:22.017Z30/30 Write block 175 len 32768 data: 13 13 12 12 9 7 8 9
140392023-09-22T23:30:22.020Z----------------------------------------------------------------
140402023-09-22T23:30:22.024Z Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:30
140412023-09-22T23:30:22.027ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
140422023-09-22T23:30:22.030Z 1 Acked 1000 Write 6 Done Done Done false
140432023-09-22T23:30:22.033Z 2 Acked 1001 Read 3 Sent Done Done false
140442023-09-22T23:30:22.036Z 3 Acked 1002 Read 3 Sent Done Done false
140452023-09-22T23:30:22.039Z 4 Acked 1003 Write 9 Sent Done Done false
140462023-09-22T23:30:22.042Z 5 Acked 1004 Read 3 Sent Done Done false
140472023-09-22T23:30:22.045Z 6 Acked 1005 Read 2 Sent Done Done false
140482023-09-22T23:30:22.048Z 7 Acked 1006 Write 9 Sent Done Done false
140492023-09-22T23:30:22.051Z 8 Acked 1007 Read 9 Sent Done Done false
140502023-09-22T23:30:22.057Z 9 Acked 1008 Read 6 Sent Done Done false
140512023-09-22T23:30:22.060Z 10 Acked 1009 Flush 0 Sent Done Done false
140522023-09-22T23:30:22.063Z 11 Acked 1010 Write 7 Sent Done Done false
140532023-09-22T23:30:22.069Z 12 Acked 1011 Read 6 Sent Done Done false
140542023-09-22T23:30:22.072Z 13 Acked 1012 Flush 0 Sent Done Done false
140552023-09-22T23:30:22.075Z 14 Acked 1013 Write 6 Sent Done Done false
140562023-09-22T23:30:22.079Z 15 Acked 1014 Write 10 Sent Done Done false
140572023-09-22T23:30:22.082Z 16 Acked 1015 Read 1 Sent Done Done false
140582023-09-22T23:30:22.085Z 17 Acked 1016 Write 3 Sent Done Done false
140592023-09-22T23:30:22.088Z 18 Acked 1017 Write 3 Sent Done Done false
140602023-09-22T23:30:22.091Z 19 Acked 1018 Write 8 Sent Done Done false
140612023-09-22T23:30:22.095Z 20 Acked 1019 Read 10 Sent Done Done false
140622023-09-22T23:30:22.098Z 21 Acked 1020 Write 2 Sent Done Done false
140632023-09-22T23:30:22.101Z 22 Acked 1021 Read 10 Sent Done Done false
140642023-09-22T23:30:22.104Z 23 Acked 1022 Write 9 Sent Done Done false
140652023-09-22T23:30:22.107Z 24 Acked 1023 Write 4 Sent Done Done false
140662023-09-22T23:30:22.110Z 25 Acked 1024 Flush 0 Sent Done Done false
140672023-09-22T23:30:22.113Z 26 Acked 1025 Write 10 Sent Done Done false
140682023-09-22T23:30:22.116Z 27 Acked 1026 Read 8 Sent Done Done false
140692023-09-22T23:30:22.119Z 28 Acked 1027 Read 6 Sent Done Done false
140702023-09-22T23:30:22.122Z 29 Acked 1028 Write 1 Sent Sent Sent false
140712023-09-22T23:30:22.125Z 30 Acked 1029 Write 8 Sent Sent Sent false
140722023-09-22T23:30:22.128ZError: Difference in extent metadata found!
140732023-09-22T23:30:22.132Z STATES DS:0 DS:1 DS:2 TOTAL
140742023-09-22T23:30:22.135Z New 0 0 0 0
140752023-09-22T23:30:22.138Z Sent 29 2 2 33
140762023-09-22T23:30:22.141Z Done 1 28 28 57
140772023-09-22T23:30:22.144Z Skipped 0 0 0 0
140782023-09-22T23:30:22.147Z Error 0 0 0 0
140792023-09-22T23:30:22.150ZLast Flush: 0 1024 1024
140802023-09-22T23:30:22.153ZDownstairs last five completed:
140812023-09-22T23:30:22.156ZUpstairs last five completed: 30 29 28 27 26 25
140822023-09-22T23:30:22.159ZWrote out file "/tmp/test_repair_verify.data"
140832023-09-22T23:30:22.162Z
140842023-09-22T23:30:22.166ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
140852023-09-22T23:30:22.169Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.460438583Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1599}
140862023-09-22T23:30:22.173Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.460731667Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1599}
140872023-09-22T23:30:22.176Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.460754335Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1599}
140882023-09-22T23:30:22.179Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.490498372Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1599}
140892023-09-22T23:30:22.182Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.490528789Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1599}
140902023-09-22T23:30:22.185Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.490547202Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1599}
140912023-09-22T23:30:22.189Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.509102422Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1599}
140922023-09-22T23:30:22.191Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.509130883Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1599}
140932023-09-22T23:30:22.195Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:21.509149341Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1599}
140942023-09-22T23:30:22.198ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
140952023-09-22T23:30:22.201Z 0 000-019 100 106 106 153 162 162 F F F <---
140962023-09-22T23:30:22.204Z 1 020-039 105 106 106 160 161 161 T F F <---
140972023-09-22T23:30:22.207Z 2 040-059 93 93 93 139 139 139 F F F
140982023-09-22T23:30:22.210Z 3 060-079 100 106 106 153 163 163 F F F <---
140992023-09-22T23:30:22.213Z 4 080-099 102 106 106 156 163 163 F F F <---
141002023-09-22T23:30:22.216Z 5 100-119 105 106 106 160 163 163 F F F <---
141012023-09-22T23:30:22.219Z 6 120-139 100 106 106 153 163 163 F F F <---
141022023-09-22T23:30:22.222Z 7 140-159 104 104 104 159 159 159 F F F
141032023-09-22T23:30:22.225Z 8 160-179 105 105 105 160 160 160 F T T <---
141042023-09-22T23:30:22.228Z 9 180-199 100 100 100 153 153 153 F T T <---
141052023-09-22T23:30:22.231Z 10 200-219 105 105 105 160 160 160 F F F
141062023-09-22T23:30:22.234Z 11 220-239 105 106 106 160 163 163 F T T <---
141072023-09-22T23:30:22.237Z 12 240-259 103 103 103 158 158 158 F F F
141082023-09-22T23:30:22.240Z 13 260-279 102 102 102 157 157 157 F F F
141092023-09-22T23:30:22.243Z 14 280-299 105 105 105 160 160 160 F F F
141102023-09-22T23:30:22.246Z 15 300-319 105 105 105 160 160 160 F F F
141112023-09-22T23:30:22.249Z 16 320-339 105 105 105 160 160 160 F F F
141122023-09-22T23:30:22.252Z 17 340-359 102 106 106 156 163 163 F F F <---
141132023-09-22T23:30:22.255Z 18 360-379 105 105 105 160 160 160 F F F
141142023-09-22T23:30:22.258Z 19 380-399 105 105 105 160 160 160 F F F
141152023-09-22T23:30:22.261Z 20 400-419 102 102 102 156 156 156 F T T <---
141162023-09-22T23:30:22.264Z 21 420-439 105 106 106 160 163 163 F T T <---
141172023-09-22T23:30:22.267Z 22 440-459 105 105 105 160 160 160 F F F
141182023-09-22T23:30:22.270Z 23 460-479 105 106 106 160 163 163 F F F <---
141192023-09-22T23:30:22.273Z 24 480-499 99 106 106 152 163 163 F F F <---
141202023-09-22T23:30:22.277Z 25 500-519 105 105 105 160 160 160 F F F
141212023-09-22T23:30:22.280Z 26 520-539 102 102 102 157 157 157 F F F
141222023-09-22T23:30:22.283Z 27 540-559 103 103 103 158 158 158 F F F
141232023-09-22T23:30:22.286Z 28 560-579 100 100 100 153 153 153 F F F
141242023-09-22T23:30:22.288Z 29 580-599 94 106 106 143 163 163 F F F <---
141252023-09-22T23:30:22.292ZMax gen: 106, Max flush: 163
141262023-09-22T23:30:22.295ZA Difference in extent metadata is expected here
141272023-09-22T23:30:22.298ZOn loop 53
141282023-09-22T23:30:22.307Z
141292023-09-22T23:30:22.311ZVerifying data now
141302023-09-22T23:30:28.120ZNo change after verify
141312023-09-22T23:30:28.123ZLoop: 53 Downstairs dump after verify (and repair):
141322023-09-22T23:30:28.127Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.119431733Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1604}
141332023-09-22T23:30:28.129Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.119817381Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1604}
141342023-09-22T23:30:28.132Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.1198605Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1604}
141352023-09-22T23:30:28.138Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.138848098Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1604}
141362023-09-22T23:30:28.141Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.138904139Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1604}
141372023-09-22T23:30:28.144Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.138981256Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1604}
141382023-09-22T23:30:28.149Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.149998436Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1604}
141392023-09-22T23:30:28.152Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.150051421Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1604}
141402023-09-22T23:30:28.155Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.150072078Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1604}
141412023-09-22T23:30:28.160ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
141422023-09-22T23:30:28.163Z 0 000-019 106 106 106 162 162 162 F F F
141432023-09-22T23:30:28.166Z 1 020-039 106 106 106 161 161 161 F F F
141442023-09-22T23:30:28.169Z 2 040-059 93 93 93 139 139 139 F F F
141452023-09-22T23:30:28.172Z 3 060-079 106 106 106 163 163 163 F F F
141462023-09-22T23:30:28.175Z 4 080-099 106 106 106 163 163 163 F F F
141472023-09-22T23:30:28.178Z 5 100-119 106 106 106 163 163 163 F F F
141482023-09-22T23:30:28.180Z 6 120-139 106 106 106 163 163 163 F F F
141492023-09-22T23:30:28.183Z 7 140-159 104 104 104 159 159 159 F F F
141502023-09-22T23:30:28.186Z 8 160-179 107 107 107 164 164 164 F F F
141512023-09-22T23:30:28.189Z 9 180-199 107 107 107 164 164 164 F F F
141522023-09-22T23:30:28.192Z 10 200-219 105 105 105 160 160 160 F F F
141532023-09-22T23:30:28.194Z 11 220-239 107 107 107 164 164 164 F F F
141542023-09-22T23:30:28.198Z 12 240-259 103 103 103 158 158 158 F F F
141552023-09-22T23:30:28.201Z 13 260-279 102 102 102 157 157 157 F F F
141562023-09-22T23:30:28.204Z 14 280-299 105 105 105 160 160 160 F F F
141572023-09-22T23:30:28.207Z 15 300-319 105 105 105 160 160 160 F F F
141582023-09-22T23:30:28.209Z 16 320-339 105 105 105 160 160 160 F F F
141592023-09-22T23:30:28.212Z 17 340-359 106 106 106 163 163 163 F F F
141602023-09-22T23:30:28.215Z 18 360-379 105 105 105 160 160 160 F F F
141612023-09-22T23:30:28.218Z 19 380-399 105 105 105 160 160 160 F F F
141622023-09-22T23:30:28.221Z 20 400-419 107 107 107 164 164 164 F F F
141632023-09-22T23:30:28.224Z 21 420-439 107 107 107 164 164 164 F F F
141642023-09-22T23:30:28.227Z 22 440-459 105 105 105 160 160 160 F F F
141652023-09-22T23:30:28.230Z 23 460-479 106 106 106 163 163 163 F F F
141662023-09-22T23:30:28.233Z 24 480-499 106 106 106 163 163 163 F F F
141672023-09-22T23:30:28.236Z 25 500-519 105 105 105 160 160 160 F F F
141682023-09-22T23:30:28.239Z 26 520-539 102 102 102 157 157 157 F F F
141692023-09-22T23:30:28.242Z 27 540-559 103 103 103 158 158 158 F F F
141702023-09-22T23:30:28.245Z 28 560-579 100 100 100 153 153 153 F F F
141712023-09-22T23:30:28.247Z 29 580-599 106 106 106 163 163 163 F F F
141722023-09-22T23:30:28.250ZMax gen: 107, Max flush: 164
141732023-09-22T23:30:28.253Z
141742023-09-22T23:30:28.256ZBegin loop 54
141752023-09-22T23:30:28.259ZDownstairs to restart: 0
141762023-09-22T23:30:28.262Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.197248206Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
141772023-09-22T23:30:28.265Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.197632869Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
141782023-09-22T23:30:28.268Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.197689497Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
141792023-09-22T23:30:28.271Z{"msg":"Crucible stats registered with UUID: 4d51830e-10e2-41c5-93c6-10c853933e68","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.19773277Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
141802023-09-22T23:30:28.274Z{"msg":"Crucible 4d51830e-10e2-41c5-93c6-10c853933e68 has session id: c7ac92df-7e97-4f24-96f1-d9844b673e9f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.19779903Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
141812023-09-22T23:30:28.278ZCrucible runtime is spawned
141822023-09-22T23:30:28.280Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.198088933Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606,"task":"up_listen"}
141832023-09-22T23:30:28.283Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30The guest has requested activation with gen:108
141842023-09-22T23:30:28.286Z,"time":"2023-09-22T23:30:28.198188484Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
141852023-09-22T23:30:28.290Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.198307853Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
141862023-09-22T23:30:28.293Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.198352869Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606,"looper":"0"}
141872023-09-22T23:30:28.296Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.198403456Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606,"looper":"1"}
141882023-09-22T23:30:28.299Z{"msg":"[1] 4d51830e-10e2-41c5-93c6-10c853933e68 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.198634191Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606,"looper":"1"}
141892023-09-22T23:30:28.302Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.198690823Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606,"looper":"2"}
141902023-09-22T23:30:28.305Z{"msg":"[2] 4d51830e-10e2-41c5-93c6-10c853933e68 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.198899325Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606,"looper":"2"}
141912023-09-22T23:30:28.308Z{"msg":"4d51830e-10e2-41c5-93c6-10c853933e68 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.199194002Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
141922023-09-22T23:30:28.310Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.199258664Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
141932023-09-22T23:30:28.313Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.199350361Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
141942023-09-22T23:30:28.316Z{"msg":"Set desired generation to :108","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.199585571Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
141952023-09-22T23:30:28.319Z{"msg":"[1] 4d51830e-10e2-41c5-93c6-10c853933e68 (a2e74d37-5d99-4897-bd84-67910e0c5d9b) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.199976069Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
141962023-09-22T23:30:28.324Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.200049326Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
141972023-09-22T23:30:28.328Z{"msg":"[1] client is_active_req TRUE, promote! session a2e74d37-5d99-4897-bd84-67910e0c5d9b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.200118838Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
141982023-09-22T23:30:28.331Z{"msg":"[2] 4d51830e-10e2-41c5-93c6-10c853933e68 (a2e74d37-5d99-4897-bd84-67910e0c5d9b) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.200176598Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
141992023-09-22T23:30:28.334Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.200206576Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142002023-09-22T23:30:28.337Z{"msg":"[2] client is_active_req TRUE, promote! session a2e74d37-5d99-4897-bd84-67910e0c5d9b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.200279528Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142012023-09-22T23:30:28.340Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.200768903Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142022023-09-22T23:30:28.343Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.200888533Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142032023-09-22T23:30:28.346Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.200934357Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142042023-09-22T23:30:28.348Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.200960537Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142052023-09-22T23:30:28.351Z{"msg":"4d51830e-10e2-41c5-93c6-10c853933e68 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.20099884Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142062023-09-22T23:30:28.354Z{"msg":"4d51830e-10e2-41c5-93c6-10c853933e68 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.201039303Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142072023-09-22T23:30:28.357Z{"msg":"[2] 4d51830e-10e2-41c5-93c6-10c853933e68 (a2e74d37-5d99-4897-bd84-67910e0c5d9b) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.203767856Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142082023-09-22T23:30:28.359Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.203825895Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142092023-09-22T23:30:28.362Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:30:28.203868741Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142102023-09-22T23:30:28.366Z{"msg":"[1] 4d51830e-10e2-41c5-93c6-10c853933e68 (a2e74d37-5d99-4897-bd84-67910e0c5d9b) New WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.203911028Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142112023-09-22T23:30:28.369Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.203950565Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142122023-09-22T23:30:28.372Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:30:28.203989049Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142132023-09-22T23:30:28.376Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.204028196Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142142023-09-22T23:30:28.379Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.204066315Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142152023-09-22T23:30:28.382Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.204105324Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142162023-09-22T23:30:28.385Z{"msg":"4d51830e-10e2-41c5-93c6-10c853933e68 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.204144373Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142172023-09-22T23:30:28.389Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.204183649Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142182023-09-22T23:30:28.392Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.204221666Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142192023-09-22T23:30:28.396Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.20426143Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142202023-09-22T23:30:28.399Z{"msg":"4d51830e-10e2-41c5-93c6-10c853933e68 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.204301523Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142212023-09-22T23:30:28.402Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.204341599Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142222023-09-22T23:30:28.405Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:28.204361053Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142232023-09-22T23:30:29.201Z{"msg":"[0] 4d51830e-10e2-41c5-93c6-10c853933e68 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.199702969Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606,"looper":"0"}
142242023-09-22T23:30:29.204Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.199794665Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142252023-09-22T23:30:29.209Z{"msg":"[0] 4d51830e-10e2-41c5-93c6-10c853933e68 (a2e74d37-5d99-4897-bd84-67910e0c5d9b) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.20098968Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142262023-09-22T23:30:29.213Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.201053376Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142272023-09-22T23:30:29.217Z{"msg":"[0] client is_active_req TRUE, promote! session a2e74d37-5d99-4897-bd84-67910e0c5d9b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.201099084Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142282023-09-22T23:30:29.220Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.201662081Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142292023-09-22T23:30:29.223Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.201723039Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142302023-09-22T23:30:29.226Z{"msg":"4d51830e-10e2-41c5-93c6-10c853933e68 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.201767893Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142312023-09-22T23:30:29.229Z{"msg":"[0] 4d51830e-10e2-41c5-93c6-10c853933e68 (a2e74d37-5d99-4897-bd84-67910e0c5d9b) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.204129185Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142322023-09-22T23:30:29.232Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.204290849Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142332023-09-22T23:30:29.235Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:30:29.204349521Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142342023-09-22T23:30:29.238Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.20440569Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142352023-09-22T23:30:29.241Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.204467835Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142362023-09-22T23:30:29.244Z{"msg":"4d51830e-10e2-41c5-93c6-10c853933e68 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.204534645Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142372023-09-22T23:30:29.247ZThe guest has finished waiting for activation with:108
142382023-09-22T23:30:29.250Z{"msg"Wait for a query_work_queue command to finish before sending IO
142392023-09-22T23:30:29.253Z:"[0]R flush_numbers[0..12]: [162, 161, 139, 163, 163, 163, 163, 159, 164, 164, 160, 164]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.204597263Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142402023-09-22T23:30:29.259Z{"msg":"[0]R generation[0..12]: [106, 106, 93, 106, 106, 106, 106, 104, 107, 107, 105, 107]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.204658113Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142412023-09-22T23:30:29.666Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.204719128Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142422023-09-22T23:30:29.670Z{"msg":"[1]R flush_numbers[0..12]: [162, 161, 139, 163, 163, 163, 163, 159, 164, 164, 160, 164]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.204774696Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142432023-09-22T23:30:29.673Z{"msg":"[1]R generation[0..12]: [106, 106, 93, 106, 106, 106, 106, 104, 107, 107, 105, 107]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.204830426Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142442023-09-22T23:30:29.677Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.204887815Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142452023-09-22T23:30:29.684Z{"msg":"[2]R flush_numbers[0..12]: [162, 161, 139, 163, 163, 163, 163, 159, 164, 164, 160, 164]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.204950072Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142462023-09-22T23:30:29.688Z{"msg":"[2]R generation[0..12]: [106, 106, 93, 106, 106, 106, 106, 104, 107, 107, 105, 107]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.205009348Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142472023-09-22T23:30:29.692Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.205072501Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142482023-09-22T23:30:29.699Z{"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.205125015Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142492023-09-22T23:30:29.703Z{"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.205179805Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142502023-09-22T23:30:29.708Z{"msg":"Next flush: 165","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.205216482Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142512023-09-22T23:30:29.711Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.20526716Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142522023-09-22T23:30:29.715Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.205320859Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142532023-09-22T23:30:29.720Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.205375223Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142542023-09-22T23:30:29.725Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
142552023-09-22T23:30:29.729Zlevel":30,"time":"2023-09-22T23:30:29.205577134Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142562023-09-22T23:30:29.747Z{"msg":"4d51830e-10e2-41c5-93c6-10c853933e68 is now active with session: a2e74d37-5d99-4897-bd84-67910e0c5d9b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.205637626Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142572023-09-22T23:30:30.167Z{"msg":"4d51830e-10e2-41c5-93c6-10c853933e68 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.205692085Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142582023-09-22T23:30:30.170Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.205744954Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142592023-09-22T23:30:30.174Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.205797918Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142602023-09-22T23:30:30.176Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.205850183Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142612023-09-22T23:30:30.179Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.205901819Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142622023-09-22T23:30:30.182Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.205954384Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142632023-09-22T23:30:30.186Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.206006511Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142642023-09-22T23:30:30.189Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.206057837Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142652023-09-22T23:30:30.192ZLoading write count information from file "/tmp/test_repair_verify.data"
142662023-09-22T23:30:30.195Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.206113998Z"Run Repair workload
142672023-09-22T23:30:30.199Z,"hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142682023-09-22T23:30:30.202Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.206227763Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142692023-09-22T23:30:30.205Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.206283202Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142702023-09-22T23:30:30.207Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.206335454Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1606}
142712023-09-22T23:30:30.211Z01/30 Write block 58 len 8192 data: 7 8
142722023-09-22T23:30:30.213Z02/30 Write block 383 len 8192 data: 7 7
142732023-09-22T23:30:30.217Z03/30 Read block 477 len 16384
142742023-09-22T23:30:30.220Z04/30 Read block 588 len 12288
142752023-09-22T23:30:30.223Z05/30 Read block 347 len 32768
142762023-09-22T23:30:30.226Z06/30 Write block 49 len 36864 data: 8 7 8 6 4 3 5 4 4
142772023-09-22T23:30:30.229Z07/30 Write block 428 len 20480 data: 11 11 12 10 12
142782023-09-22T23:30:30.232Z08/30 Write block 525 len 16384 data: 10 11 11 10
142792023-09-22T23:30:30.235Z09/30 Read block 170 len 36864
142802023-09-22T23:30:30.239Z10/30 Read block 519 len 40960
142812023-09-22T23:30:30.242Z11/30 Write block 483 len 28672 data: 8 12 12 14 11 15 16
142822023-09-22T23:30:30.245Z12/30 Read block 561 len 40960
142832023-09-22T23:30:30.248Z13/30 Write block 402 len 36864 data: 12 10 10 9 8 8 7 9 7
142842023-09-22T23:30:30.251Z14/30 Read block 13 len 40960
142852023-09-22T23:30:30.254Z15/30 Read block 333 len 20480
142862023-09-22T23:30:30.257Z16/30 Write block 389 len 28672 data: 9 9 8 8 6 8 10
142872023-09-22T23:30:30.260Z17/30 Read block 86 len 16384
142882023-09-22T23:30:30.263Z18/30 Write block 585 len 16384 data: 6 4 4 6
142892023-09-22T23:30:30.266Z19/30 Read block 240 len 20480
142902023-09-22T23:30:30.269Z20/30 Write block 430 len 16384 data: 13 11 13 11
142912023-09-22T23:30:30.272Z21/30 Write block 444 len 40960 data: 13 12 9 7 9 10 11 10 10 10
142922023-09-22T23:30:30.275Z22/30 Write block 350 len 24576 data: 5 7 6 5 6 7
142932023-09-22T23:30:30.278Z23/30 Write block 54 len 4096 data: 4
142942023-09-22T23:30:30.281Z24/30 Read block 53 len 8192
142952023-09-22T23:30:30.284Z25/30 Write block 531 len 40960 data: 8 9 11 9 10 10 12 13 13 8
142962023-09-22T23:30:30.288Z26/30 Read block 367 len 24576
142972023-09-22T23:30:30.291Z27/30 Write block 307 len 32768 data: 10 9 7 5 4 5 7 9
142982023-09-22T23:30:30.294Z28/30 Read block 504 len 24576
142992023-09-22T23:30:30.297Z29/30 Write block 155 len 20480 data: 7 6 8 11 13
143002023-09-22T23:30:30.300Z30/30 Read block 137 len 16384
143012023-09-22T23:30:30.303Z----------------------------------------------------------------
143022023-09-22T23:30:30.306Z Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30
143032023-09-22T23:30:30.310ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
143042023-09-22T23:30:30.312Z 1 Acked 1000 Write 2 Done Done Done false
143052023-09-22T23:30:30.316Z 2 Acked 1001 Write 2 Sent Done Done false
143062023-09-22T23:30:30.319ZError: Difference in extent metadata found!
143072023-09-22T23:30:30.322Z 3 Acked 1002 Read 4 Sent Done Done false
143082023-09-22T23:30:30.325Z 4 Acked 1003 Read 3 Sent Done Done false
143092023-09-22T23:30:30.328Z 5 Acked 1004 Read 8 Sent Done Done false
143102023-09-22T23:30:30.331Z 6 Acked 1005 Write 9 Sent Done Done false
143112023-09-22T23:30:30.334Z 7 Acked 1006 Write 5 Sent Done Done false
143122023-09-22T23:30:30.337Z 8 Acked 1007 Write 4 Sent Done Done false
143132023-09-22T23:30:30.340Z 9 Acked 1008 Read 9 Sent Done Done false
143142023-09-22T23:30:30.343Z 10 Acked 1009 Read 10 Sent Done Done false
143152023-09-22T23:30:30.347Z 11 Acked 1010 Write 7 Sent Done Done false
143162023-09-22T23:30:30.350Z 12 Acked 1011 Read 10 Sent Done Done false
143172023-09-22T23:30:30.353Z 13 Acked 1012 Write 9 Sent Done Done false
143182023-09-22T23:30:30.356Z 14 Acked 1013 Read 10 Sent Done Done false
143192023-09-22T23:30:30.358Z 15 Acked 1014 Read 5 Sent Done Done false
143202023-09-22T23:30:30.362Z 16 Acked 1015 Write 7 Sent Done Done false
143212023-09-22T23:30:30.365Z 17 Acked 1016 Read 4 Sent Done Done false
143222023-09-22T23:30:30.368Z 18 Acked 1017 Write 4 Sent Done Done false
143232023-09-22T23:30:30.371Z 19 Acked 1018 Read 5 Sent Done Done false
143242023-09-22T23:30:30.374Z 20 Acked 1019 Write 4 Sent Done Done false
143252023-09-22T23:30:30.377Z 21 Acked 1020 Write 10 Sent Done Done false
143262023-09-22T23:30:30.379Z 22 Acked 1021 Write 6 Sent Done Done false
143272023-09-22T23:30:30.383Z 23 Acked 1022 Write 1 Sent Done Done false
143282023-09-22T23:30:30.386Z 24 Acked 1023 Read 2 Sent Done Done false
143292023-09-22T23:30:30.389Z 25 Acked 1024 Write 10 Sent Done Done false
143302023-09-22T23:30:30.392Z 26 Acked 1025 Read 6 Sent Done Done false
143312023-09-22T23:30:30.395Z 27 Acked 1026 Write 8 Sent Done Done false
143322023-09-22T23:30:30.399Z 28 Acked 1027 Read 6 Sent Done Done false
143332023-09-22T23:30:30.402Z 29 Acked 1028 Write 5 Sent Sent Done false
143342023-09-22T23:30:30.406Z 30 Acked 1029 Read 4 Sent Sent Done false
143352023-09-22T23:30:30.409Z STATES DS:0 DS:1 DS:2 TOTAL
143362023-09-22T23:30:30.412Z New 0 0 0 0
143372023-09-22T23:30:30.415Z Sent 29 2 0 31
143382023-09-22T23:30:30.419Z Done 1 28 30 59
143392023-09-22T23:30:30.422Z Skipped 0 0 0 0
143402023-09-22T23:30:30.426Z Error 0 0 0 0
143412023-09-22T23:30:30.430ZLast Flush: 0 0 0
143422023-09-22T23:30:30.433ZDownstairs last five completed:
143432023-09-22T23:30:30.436ZUpstairs last five completed: 30 29 28 27 26 25
143442023-09-22T23:30:30.439ZWrote out file "/tmp/test_repair_verify.data"
143452023-09-22T23:30:30.442Z
143462023-09-22T23:30:30.447ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
143472023-09-22T23:30:30.450Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.342762002Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1607}
143482023-09-22T23:30:30.453Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.343030744Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1607}
143492023-09-22T23:30:30.460Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.343053204Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1607}
143502023-09-22T23:30:30.463Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.370350775Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1607}
143512023-09-22T23:30:30.466Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.370381048Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1607}
143522023-09-22T23:30:30.469Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.370399277Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1607}
143532023-09-22T23:30:30.471Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.399389085Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1607}
143542023-09-22T23:30:30.475Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.399419254Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1607}
143552023-09-22T23:30:30.478Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:29.399437816Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1607}
143562023-09-22T23:30:30.481ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
143572023-09-22T23:30:30.484Z 0 000-019 106 106 106 162 162 162 F F F
143582023-09-22T23:30:30.487Z 1 020-039 106 106 106 161 161 161 F F F
143592023-09-22T23:30:30.490Z 2 040-059 93 93 93 139 139 139 T T T
143602023-09-22T23:30:30.493Z 3 060-079 106 106 106 163 163 163 F F F
143612023-09-22T23:30:30.496Z 4 080-099 106 106 106 163 163 163 F F F
143622023-09-22T23:30:30.499Z 5 100-119 106 106 106 163 163 163 F F F
143632023-09-22T23:30:30.502Z 6 120-139 106 106 106 163 163 163 F F F
143642023-09-22T23:30:30.505Z 7 140-159 104 104 104 159 159 159 F T T <---
143652023-09-22T23:30:30.508Z 8 160-179 107 107 107 164 164 164 F F F
143662023-09-22T23:30:30.511Z 9 180-199 107 107 107 164 164 164 F F F
143672023-09-22T23:30:30.514Z 10 200-219 105 105 105 160 160 160 F F F
143682023-09-22T23:30:30.517Z 11 220-239 107 107 107 164 164 164 F F F
143692023-09-22T23:30:30.520Z 12 240-259 103 103 103 158 158 158 F F F
143702023-09-22T23:30:30.523Z 13 260-279 102 102 102 157 157 157 F F F
143712023-09-22T23:30:30.526Z 14 280-299 105 105 105 160 160 160 F F F
143722023-09-22T23:30:30.529Z 15 300-319 105 105 105 160 160 160 F T T <---
143732023-09-22T23:30:30.532Z 16 320-339 105 105 105 160 160 160 F F F
143742023-09-22T23:30:30.535Z 17 340-359 106 106 106 163 163 163 F T T <---
143752023-09-22T23:30:30.538Z 18 360-379 105 105 105 160 160 160 F F F
143762023-09-22T23:30:30.541Z 19 380-399 105 105 105 160 160 160 F T T <---
143772023-09-22T23:30:30.544Z 20 400-419 107 107 107 164 164 164 F T T <---
143782023-09-22T23:30:30.547Z 21 420-439 107 107 107 164 164 164 F T T <---
143792023-09-22T23:30:30.550Z 22 440-459 105 105 105 160 160 160 F T T <---
143802023-09-22T23:30:30.554Z 23 460-479 106 106 106 163 163 163 F F F
143812023-09-22T23:30:30.557Z 24 480-499 106 106 106 163 163 163 F T T <---
143822023-09-22T23:30:30.560Z 25 500-519 105 105 105 160 160 160 F F F
143832023-09-22T23:30:30.563Z 26 520-539 102 102 102 157 157 157 F T T <---
143842023-09-22T23:30:30.567Z 27 540-559 103 103 103 158 158 158 F T T <---
143852023-09-22T23:30:30.570Z 28 560-579 100 100 100 153 153 153 F F F
143862023-09-22T23:30:30.573Z 29 580-599 106 106 106 163 163 163 F T T <---
143872023-09-22T23:30:30.576ZMax gen: 107, Max flush: 164
143882023-09-22T23:30:30.579ZA Difference in extent metadata is expected here
143892023-09-22T23:30:30.582ZOn loop 54
143902023-09-22T23:30:30.585Z
143912023-09-22T23:30:30.588ZVerifying data now
143922023-09-22T23:30:35.888ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
143932023-09-22T23:30:35.892Z51a52
143942023-09-22T23:30:35.895Z> 8,
143952023-09-22T23:30:35.898Z52a54
143962023-09-22T23:30:35.901Z> 8,
143972023-09-22T23:30:35.904Z54c56,57
143982023-09-22T23:30:35.907Z< 7,
143992023-09-22T23:30:35.910Z---
144002023-09-22T23:30:35.913Z> 4,
144012023-09-22T23:30:35.916Z> 4,
144022023-09-22T23:30:35.918Z56,57d58
144032023-09-22T23:30:35.921Z< 3,
144042023-09-22T23:30:35.924Z< 2,
144052023-09-22T23:30:35.927Z59,60c60
144062023-09-22T23:30:35.930Z< 3,
144072023-09-22T23:30:35.933Z< 3,
144082023-09-22T23:30:35.937Z---
144092023-09-22T23:30:35.939Z> 4,
144102023-09-22T23:30:35.942Zdiff found after verify
144112023-09-22T23:30:35.945ZLoop: 54 Downstairs dump after verify (and repair):
144122023-09-22T23:30:35.948Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.916503854Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1615}
144132023-09-22T23:30:35.951Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.916831828Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1615}
144142023-09-22T23:30:35.954Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.916875202Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1615}
144152023-09-22T23:30:35.957Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.935309116Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1615}
144162023-09-22T23:30:35.960Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.935362765Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1615}
144172023-09-22T23:30:35.963Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.935401356Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1615}
144182023-09-22T23:30:35.966Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.946102111Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1615}
144192023-09-22T23:30:35.969Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.94615424Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1615}
144202023-09-22T23:30:35.972Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.946193114Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1615}
144212023-09-22T23:30:35.975ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
144222023-09-22T23:30:35.978Z 0 000-019 106 106 106 162 162 162 F F F
144232023-09-22T23:30:35.981Z 1 020-039 106 106 106 161 161 161 F F F
144242023-09-22T23:30:35.984Z 2 040-059 108 108 108 165 165 165 F F F
144252023-09-22T23:30:35.987Z 3 060-079 106 106 106 163 163 163 F F F
144262023-09-22T23:30:35.990Z 4 080-099 106 106 106 163 163 163 F F F
144272023-09-22T23:30:35.993Z 5 100-119 106 106 106 163 163 163 F F F
144282023-09-22T23:30:35.996Z 6 120-139 106 106 106 163 163 163 F F F
144292023-09-22T23:30:36.000Z 7 140-159 108 108 108 165 165 165 F F F
144302023-09-22T23:30:36.003Z 8 160-179 107 107 107 164 164 164 F F F
144312023-09-22T23:30:36.006Z 9 180-199 107 107 107 164 164 164 F F F
144322023-09-22T23:30:36.009Z 10 200-219 105 105 105 160 160 160 F F F
144332023-09-22T23:30:36.012Z 11 220-239 107 107 107 164 164 164 F F F
144342023-09-22T23:30:36.015Z 12 240-259 103 103 103 158 158 158 F F F
144352023-09-22T23:30:36.018Z 13 260-279 102 102 102 157 157 157 F F F
144362023-09-22T23:30:36.021Z 14 280-299 105 105 105 160 160 160 F F F
144372023-09-22T23:30:36.024Z 15 300-319 108 108 108 165 165 165 F F F
144382023-09-22T23:30:36.027Z 16 320-339 105 105 105 160 160 160 F F F
144392023-09-22T23:30:36.030Z 17 340-359 108 108 108 165 165 165 F F F
144402023-09-22T23:30:36.035Z 18 360-379 105 105 105 160 160 160 F F F
144412023-09-22T23:30:36.038Z 19 380-399 108 108 108 165 165 165 F F F
144422023-09-22T23:30:36.041Z 20 400-419 108 108 108 165 165 165 F F F
144432023-09-22T23:30:36.044Z 21 420-439 108 108 108 165 165 165 F F F
144442023-09-22T23:30:36.046Z 22 440-459 108 108 108 165 165 165 F F F
144452023-09-22T23:30:36.050Z 23 460-479 106 106 106 163 163 163 F F F
144462023-09-22T23:30:36.053Z 24 480-499 108 108 108 165 165 165 F F F
144472023-09-22T23:30:36.056Z 25 500-519 105 105 105 160 160 160 F F F
144482023-09-22T23:30:36.059Z 26 520-539 108 108 108 165 165 165 F F F
144492023-09-22T23:30:36.062Z 27 540-559 108 108 108 165 165 165 F F F
144502023-09-22T23:30:36.065Z 28 560-579 100 100 100 153 153 153 F F F
144512023-09-22T23:30:36.067Z 29 580-599 108 108 108 165 165 165 F F F
144522023-09-22T23:30:36.070ZMax gen: 108, Max flush: 165
144532023-09-22T23:30:36.076Z
144542023-09-22T23:30:36.079ZBegin loop 55
144552023-09-22T23:30:36.082ZDownstairs to restart: 2
144562023-09-22T23:30:36.085Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.995050519Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144572023-09-22T23:30:36.088Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.995416533Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144582023-09-22T23:30:36.091Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.995474171Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144592023-09-22T23:30:36.094Z{"msg":"Crucible stats registered with UUID: 8cd456a6-fc82-4741-b20b-fcad6b2fad3e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.995499397Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144602023-09-22T23:30:36.097Z{"msg":"Crucible 8cd456a6-fc82-4741-b20b-fcad6b2fad3e has session id: 79287abe-ae3d-4a6e-98b7-9f7c22570b41","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.99554765Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144612023-09-22T23:30:36.100ZCrucible runtime is spawned
144622023-09-22T23:30:36.103ZThe guest has requested activation with gen:110
144632023-09-22T23:30:36.106Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.995875861Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617,"task":"up_listen"}
144642023-09-22T23:30:36.109Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.995956745Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617,"looper":"1"}
144652023-09-22T23:30:36.112Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.996013223Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144662023-09-22T23:30:36.115Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.996054228Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144672023-09-22T23:30:36.118Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.996189767Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617,"looper":"2"}
144682023-09-22T23:30:36.122Z{"msg":"[1] 8cd456a6-fc82-4741-b20b-fcad6b2fad3e looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.996274767Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617,"looper":"1"}
144692023-09-22T23:30:36.125Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.996341595Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617,"looper":"0"}
144702023-09-22T23:30:36.128Z{"msg":"[0] 8cd456a6-fc82-4741-b20b-fcad6b2fad3e looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.996510515Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617,"looper":"0"}
144712023-09-22T23:30:36.131Z{"msg":"8cd456a6-fc82-4741-b20b-fcad6b2fad3e active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.996740407Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144722023-09-22T23:30:36.133Z{"msg":"Set desired generation to :110","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.996791748Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144732023-09-22T23:30:36.136Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.996928676Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144742023-09-22T23:30:36.139Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.997261368Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144752023-09-22T23:30:36.147Z{"msg":"[1] 8cd456a6-fc82-4741-b20b-fcad6b2fad3e (088ff428-bf60-48b5-99e2-e6155f90e6e5) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.997624351Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144762023-09-22T23:30:36.151Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.997695246Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144772023-09-22T23:30:36.154Z{"msg":"[1] client is_active_req TRUE, promote! session 088ff428-bf60-48b5-99e2-e6155f90e6e5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.9977675Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144782023-09-22T23:30:36.157Z{"msg":"[0] 8cd456a6-fc82-4741-b20b-fcad6b2fad3e (088ff428-bf60-48b5-99e2-e6155f90e6e5) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.997851635Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144792023-09-22T23:30:36.160Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.997920849Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144802023-09-22T23:30:36.163Z{"msg":"[0] client is_active_req TRUE, promote! session 088ff428-bf60-48b5-99e2-e6155f90e6e5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.998153309Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144812023-09-22T23:30:36.166Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.998386005Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144822023-09-22T23:30:36.169Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.998482152Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144832023-09-22T23:30:36.171Z{"msg":"8cd456a6-fc82-4741-b20b-fcad6b2fad3e WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.99852671Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144842023-09-22T23:30:36.174Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.998638682Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144852023-09-22T23:30:36.177Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.998694869Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144862023-09-22T23:30:36.180Z{"msg":"8cd456a6-fc82-4741-b20b-fcad6b2fad3e WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:35.99874455Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144872023-09-22T23:30:36.183Z{"msg":"[1] 8cd456a6-fc82-4741-b20b-fcad6b2fad3e (088ff428-bf60-48b5-99e2-e6155f90e6e5) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:36.000683898Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144882023-09-22T23:30:36.186Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:36.000789895Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144892023-09-22T23:30:36.190Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:30:36.000860013Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144902023-09-22T23:30:36.193Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:36.000905226Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144912023-09-22T23:30:36.196Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:36.000945Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144922023-09-22T23:30:36.199Z{"msg":"8cd456a6-fc82-4741-b20b-fcad6b2fad3e WaitActive WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:36.000984017Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144932023-09-22T23:30:36.202Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:36.001022724Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144942023-09-22T23:30:36.205Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:36.001060217Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144952023-09-22T23:30:36.208Z{"msg":"[0] 8cd456a6-fc82-4741-b20b-fcad6b2fad3e (088ff428-bf60-48b5-99e2-e6155f90e6e5) WaitActive WaitQuorum New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:36.001990406Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144962023-09-22T23:30:36.211Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:36.002030277Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144972023-09-22T23:30:36.214Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:30:36.002062333Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144982023-09-22T23:30:36.217Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:36.002082314Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
144992023-09-22T23:30:36.220Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:36.002100756Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145002023-09-22T23:30:36.224Z{"msg":"8cd456a6-fc82-4741-b20b-fcad6b2fad3e WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:36.002119197Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145012023-09-22T23:30:36.226Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:36.002137175Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145022023-09-22T23:30:36.230Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:36.002155036Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145032023-09-22T23:30:36.998Z{"msg":"[2] 8cd456a6-fc82-4741-b20b-fcad6b2fad3e looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:36.997982733Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617,"looper":"2"}
145042023-09-22T23:30:37.002Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:36.998078632Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145052023-09-22T23:30:37.005Z{"msg":"[2] 8cd456a6-fc82-4741-b20b-fcad6b2fad3e (088ff428-bf60-48b5-99e2-e6155f90e6e5) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:36.999213623Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145062023-09-22T23:30:37.008Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:36.999270561Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145072023-09-22T23:30:37.011Z{"msg":"[2] client is_active_req TRUE, promote! session 088ff428-bf60-48b5-99e2-e6155f90e6e5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:36.999315821Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145082023-09-22T23:30:37.014Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:36.9998895Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145092023-09-22T23:30:37.017Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:36.999950221Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145102023-09-22T23:30:37.020Z{"msg":"8cd456a6-fc82-4741-b20b-fcad6b2fad3e WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:36.999994553Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145112023-09-22T23:30:37.023Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.002950112Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145122023-09-22T23:30:37.026Z{"msg":"[2] 8cd456a6-fc82-4741-b20b-fcad6b2fad3e (088ff428-bf60-48b5-99e2-e6155f90e6e5) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.003096129Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145132023-09-22T23:30:37.029Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.003200605Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145142023-09-22T23:30:37.033Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:30:37.003278822Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145152023-09-22T23:30:37.036Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.003344593Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145162023-09-22T23:30:37.039Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.003411392Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145172023-09-22T23:30:37.042Z{"msg":"8cd456a6-fc82-4741-b20b-fcad6b2fad3e WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.003477218Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145182023-09-22T23:30:37.046Z{"msg":"[0]R flush_numbers[0..12]: [162, 161, 165, 163, 163, 163, 163, 165, 164, 164, 160, 164]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.003542778Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145192023-09-22T23:30:37.049Z{"msg":"[0]R generation[0..12]: [106, 106, 108, 106, 106, 106, 106, 108, 107, 107, 105, 107]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.003649459Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145202023-09-22T23:30:37.052Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.003754225Z","The guest has finished waiting for activation with:110
145212023-09-22T23:30:37.055ZWait for a query_work_queue command to finish before sending IO
145222023-09-22T23:30:37.058Zhostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145232023-09-22T23:30:37.061Z{"msg":"[1]R flush_numbers[0..12]: [162, 161, 165, 163, 163, 163, 163, 165, 164, 164, 160, 164]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.003946261Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145242023-09-22T23:30:37.064Z{"msg":"[1]R generation[0..12]: [106, 106, 108, 106, 106, 106, 106, 108, 107, 107, 105, 107]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.004002884Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145252023-09-22T23:30:37.068Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.004046297Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145262023-09-22T23:30:37.072Z{"msg":"[2]R flush_numbers[0..12]: [162, 161, 165, 163, 163, 163, 163, 165, 164, 164, 160, 164]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.004087267Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145272023-09-22T23:30:37.075Z{"msg":"[2]R generation[0..12]: [106, 106, 108, 106, 106, 106, 106, 108, 107, 107, 105, 107]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.004109848Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145282023-09-22T23:30:37.078Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.004156917Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145292023-09-22T23:30:37.081Z{"msg":"Max found gen is 109","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.004176225Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145302023-09-22T23:30:37.084Z{"msg":"Generation requested: 110 >= found:109","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.004217489Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145312023-09-22T23:30:37.088Z{"msg":"Next flush: 166","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.0042569Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145322023-09-22T23:30:37.091Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.004295161Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145332023-09-22T23:30:37.094Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.004314831Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145342023-09-22T23:30:37.097Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.004332589Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145352023-09-22T23:30:37.101Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.004351784Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145362023-09-22T23:30:37.105Z{"msg":"8cd456a6-fc82-4741-b20b-fcad6b2fad3e is now active with session: 088ff428-bf60-48b5-99e2-e6155f90e6e5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.00439062Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145372023-09-22T23:30:37.108Z{"msg":"8cd456a6-fc82-4741-b20b-fcad6b2fad3e Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.004430486Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145382023-09-22T23:30:37.111Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.004452301Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145392023-09-22T23:30:37.114Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.004488432Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145402023-09-22T23:30:37.117Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.004527778Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145412023-09-22T23:30:37.120Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.004573806Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145422023-09-22T23:30:37.124Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.004612504Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145432023-09-22T23:30:37.127ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
145442023-09-22T23:30:37.130Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.004636846Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145452023-09-22T23:30:37.133Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.004673722Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145462023-09-22T23:30:37.136Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.004712529Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145472023-09-22T23:30:37.139Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.004750425Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145482023-09-22T23:30:37.142Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.004770101Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145492023-09-22T23:30:37.153Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.004805508Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1617}
145502023-09-22T23:30:37.156ZLoading write count information from file "/tmp/test_repair_verify.data"
145512023-09-22T23:30:37.159ZRun Repair workload
145522023-09-22T23:30:37.163Z01/30 Write block 560 len 32768 data: 8 6 5 6 9 9 10 12
145532023-09-22T23:30:37.166Z02/30 Read block 46 len 36864
145542023-09-22T23:30:37.169Z03/30 Write block 182 len 40960 data: 10 7 9 7 8 7 6 7 9 9
145552023-09-22T23:30:37.172Z04/30 Read block 574 len 16384
145562023-09-22T23:30:37.175Z05/30 Read block 416 len 36864
145572023-09-22T23:30:37.178Z06/30 Read block 32 len 32768
145582023-09-22T23:30:37.181Z07/30 Read block 214 len 24576
145592023-09-22T23:30:37.184Z08/30 Write block 292 len 4096 data: 13
145602023-09-22T23:30:37.187Z09/30 Write block 22 len 4096 data: 6
145612023-09-22T23:30:37.190Z10/30 Read block 534 len 32768
145622023-09-22T23:30:37.194Z11/30 Write block 291 len 40960 data: 11 14 12 10 11 10 7 8 9 7
145632023-09-22T23:30:37.197Z12/30 Read block 418 len 24576
145642023-09-22T23:30:37.200Z13/30 Read block 503 len 28672
145652023-09-22T23:30:37.205Z14/30 Write block 426 len 28672 data: 8 10 12 12 14 12 14
145662023-09-22T23:30:37.209Z15/30 Write block 352 len 36864 data: 7 6 7 8 6 5 7 9 10
145672023-09-22T23:30:37.212Z16/30 Flush
145682023-09-22T23:30:37.215Z17/30 Write block 412 len 32768 data: 7 8 10 10 11 11 11 12
145692023-09-22T23:30:37.218Z18/30 Write block 119 len 24576 data: 7 8 8 10 11 9
145702023-09-22T23:30:37.221Z19/30 Read block 92 len 20480
145712023-09-22T23:30:37.224Z20/30 Read block 27 len 8192
145722023-09-22T23:30:37.229Z21/30 Read block 126 len 16384
145732023-09-22T23:30:37.233Z22/30 Read block 440 len 28672
145742023-09-22T23:30:37.236Z23/30 Write block 240 len 12288 data: 12 13 10
145752023-09-22T23:30:37.239Z24/30 Write block 117 len 24576 data: 9 8 8 9 9 11
145762023-09-22T23:30:37.242Z25/30 Read block 168 len 32768
145772023-09-22T23:30:37.246Z26/30 Flush
145782023-09-22T23:30:37.249Z27/30 Write block 20 len 16384 data: 6 3 7 5
145792023-09-22T23:30:37.252Z28/30 Read block 362 len 40960
145802023-09-22T23:30:37.255Z29/30 Read block 119 len 36864
145812023-09-22T23:30:37.259Z30/30 Write block 185 len 16384 data: 8 9 8 7
145822023-09-22T23:30:37.262Z----------------------------------------------------------------
145832023-09-22T23:30:37.265Z Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30
145842023-09-22T23:30:37.268ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
145852023-09-22T23:30:37.271Z 1 Acked 1000 Write 8 Done Done Done false
145862023-09-22T23:30:37.274Z 2 Acked 1001 Read 9 Done Done Done false
145872023-09-22T23:30:37.277Z 3 Acked 1002 Write 10 Done Done Sent false
145882023-09-22T23:30:37.280Z 4 Acked 1003 Read 4 Done Done Sent false
145892023-09-22T23:30:37.283Z 5 Acked 1004 Read 9 Done Done Sent false
145902023-09-22T23:30:37.286Z 6 Acked 1005 Read 8 Done Done Sent false
145912023-09-22T23:30:37.289Z 7 Acked 1006 Read 6 Done Done Sent false
145922023-09-22T23:30:37.292Z 8 Acked 1007 Write 1 Done Done Sent false
145932023-09-22T23:30:37.296Z 9 Acked 1008 Write 1 Done Done Sent false
145942023-09-22T23:30:37.299Z 10 Acked 1009 Read 8 Done Done Sent false
145952023-09-22T23:30:37.302Z 11 Acked 1010 Write 10 Done Done Sent false
145962023-09-22T23:30:37.305Z 12 Acked 1011 Read 6 Done Done Sent false
145972023-09-22T23:30:37.308Z 13 Acked 1012 Read 7 Done Done Sent false
145982023-09-22T23:30:37.312Z 14 Acked 1013 Write 7 Done Done Sent false
145992023-09-22T23:30:37.315Z 15 Acked 1014 Write 9 Done Done Sent false
146002023-09-22T23:30:37.318Z 16 Acked 1015 Flush 0 Done Done Sent false
146012023-09-22T23:30:37.321Z 17 Acked 1016 Write 8 Done Done Sent false
146022023-09-22T23:30:37.325Z 18 Acked 1017 Write 6 Done Done Sent false
146032023-09-22T23:30:37.328Z 19 Acked 1018 Read 5 Done Done Sent false
146042023-09-22T23:30:37.331Z 20 Acked 1019 Read 2 Done Done Sent false
146052023-09-22T23:30:37.334Z 21 Acked 1020 Read 4 Done Done Sent false
146062023-09-22T23:30:37.338Z 22 Acked 1021 Read 7 Done Done Sent false
146072023-09-22T23:30:37.341Z 23 Acked 1022 Write 3 Done Done Sent false
146082023-09-22T23:30:37.374Z 24 Acked 1023 Write 6 Done Done Sent false
146092023-09-22T23:30:37.377Z 25 Acked 1024 Read 8 Done Done Sent false
146102023-09-22T23:30:37.380Z 26 Acked 1025 Flush 0 Done Done Sent false
146112023-09-22T23:30:37.383Z 27 Acked 1026 Write 4 Done Done Sent false
146122023-09-22T23:30:37.386Z 28 Acked 1027 Read 10 Done Done Sent false
146132023-09-22T23:30:37.389Z 29 Acked 1028 Read 9 Done Done Sent false
146142023-09-22T23:30:37.393Z 30 Acked 1029 Write 4 Sent Sent Sent false
146152023-09-22T23:30:37.405Z STATES DS:0 DS:1 DS:2 TOTAL
146162023-09-22T23:30:37.408Z New 0 0 0 0
146172023-09-22T23:30:37.411Z Sent 1 1 28 30
146182023-09-22T23:30:37.414Z Done 29 29 2 60
146192023-09-22T23:30:37.418Z Skipped 0 0 0 0
146202023-09-22T23:30:37.421Z Error 0 0 0 0
146212023-09-22T23:30:37.425ZLast Flush: 1025 1025 0
146222023-09-22T23:30:37.428ZDownstairs last five completed:
146232023-09-22T23:30:37.431ZUpstairs last five completed: 30 29 28 27 26 25
146242023-09-22T23:30:37.435ZWrote out file "/tmp/test_repair_verify.data"
146252023-09-22T23:30:37.438Z
146262023-09-22T23:30:37.441ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
146272023-09-22T23:30:37.444Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.153560016Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1618}
146282023-09-22T23:30:37.447Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.153931002Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1618}
146292023-09-22T23:30:37.450Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.153954928Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1618}
146302023-09-22T23:30:37.454Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.176678927Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1618}
146312023-09-22T23:30:37.458Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.176712258Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1618}
146322023-09-22T23:30:37.467ZError: Difference in extent metadata found!
146332023-09-22T23:30:37.470Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.176731163Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1618}
146342023-09-22T23:30:37.474Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.189833704Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1618}
146352023-09-22T23:30:37.477Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.189865139Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1618}
146362023-09-22T23:30:37.482Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:37.189884428Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1618}
146372023-09-22T23:30:37.485ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
146382023-09-22T23:30:37.488Z 0 000-019 106 106 106 162 162 162 F F F
146392023-09-22T23:30:37.491Z 1 020-039 110 110 106 166 166 161 T T F <---
146402023-09-22T23:30:37.493Z 2 040-059 108 108 108 165 165 165 F F F
146412023-09-22T23:30:37.497Z 3 060-079 106 106 106 163 163 163 F F F
146422023-09-22T23:30:37.500Z 4 080-099 106 106 106 163 163 163 F F F
146432023-09-22T23:30:37.503Z 5 100-119 110 110 106 167 167 163 F F F <---
146442023-09-22T23:30:37.506Z 6 120-139 110 110 106 167 167 163 F F F <---
146452023-09-22T23:30:37.508Z 7 140-159 108 108 108 165 165 165 F F F
146462023-09-22T23:30:37.512Z 8 160-179 107 107 107 164 164 164 F F F
146472023-09-22T23:30:37.516Z 9 180-199 110 110 107 166 166 164 T T F <---
146482023-09-22T23:30:37.519Z 10 200-219 105 105 105 160 160 160 F F F
146492023-09-22T23:30:37.522Z 11 220-239 107 107 107 164 164 164 F F F
146502023-09-22T23:30:37.525Z 12 240-259 110 110 103 167 167 158 F F F <---
146512023-09-22T23:30:37.528Z 13 260-279 102 102 102 157 157 157 F F F
146522023-09-22T23:30:37.531Z 14 280-299 110 110 105 166 166 160 F F F <---
146532023-09-22T23:30:37.535Z 15 300-319 110 110 108 166 166 165 F F F <---
146542023-09-22T23:30:37.538Z 16 320-339 105 105 105 160 160 160 F F F
146552023-09-22T23:30:37.542Z 17 340-359 110 110 108 166 166 165 F F F <---
146562023-09-22T23:30:37.544Z 18 360-379 110 110 105 166 166 160 F F F <---
146572023-09-22T23:30:37.547Z 19 380-399 108 108 108 165 165 165 F F F
146582023-09-22T23:30:37.550Z 20 400-419 110 110 108 167 167 165 F F F <---
146592023-09-22T23:30:37.553Z 21 420-439 110 110 108 166 166 165 F F F <---
146602023-09-22T23:30:37.556Z 22 440-459 108 108 108 165 165 165 F F F
146612023-09-22T23:30:37.559Z 23 460-479 106 106 106 163 163 163 F F F
146622023-09-22T23:30:37.562Z 24 480-499 108 108 108 165 165 165 F F F
146632023-09-22T23:30:37.566Z 25 500-519 105 105 105 160 160 160 F F F
146642023-09-22T23:30:37.569Z 26 520-539 108 108 108 165 165 165 F F F
146652023-09-22T23:30:37.572Z 27 540-559 108 108 108 165 165 165 F F F
146662023-09-22T23:30:37.575Z 28 560-579 110 110 100 166 166 153 F F T <---
146672023-09-22T23:30:37.578Z 29 580-599 108 108 108 165 165 165 F F F
146682023-09-22T23:30:37.580ZMax gen: 110, Max flush: 167
146692023-09-22T23:30:37.583ZA Difference in extent metadata is expected here
146702023-09-22T23:30:37.588ZOn loop 55
146712023-09-22T23:30:37.592Z
146722023-09-22T23:30:37.595ZVerifying data now
146732023-09-22T23:30:43.532ZNo change after verify
146742023-09-22T23:30:43.535ZLoop: 55 Downstairs dump after verify (and repair):
146752023-09-22T23:30:43.579Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.549998989Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1623}
146762023-09-22T23:30:43.582Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.550339217Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1623}
146772023-09-22T23:30:43.585Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.550364581Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1623}
146782023-09-22T23:30:43.589Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.568825565Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1623}
146792023-09-22T23:30:43.592Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.568874844Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1623}
146802023-09-22T23:30:43.595Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.56891239Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1623}
146812023-09-22T23:30:43.598Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.579669445Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1623}
146822023-09-22T23:30:43.600Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.579715866Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1623}
146832023-09-22T23:30:43.605Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.579753019Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1623}
146842023-09-22T23:30:43.609ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
146852023-09-22T23:30:43.612Z 0 000-019 106 106 106 162 162 162 F F F
146862023-09-22T23:30:43.615Z 1 020-039 111 111 111 168 168 168 F F F
146872023-09-22T23:30:43.621Z 2 040-059 108 108 108 165 165 165 F F F
146882023-09-22T23:30:43.625Z 3 060-079 106 106 106 163 163 163 F F F
146892023-09-22T23:30:43.628Z 4 080-099 106 106 106 163 163 163 F F F
146902023-09-22T23:30:43.632Z 5 100-119 110 110 110 167 167 167 F F F
146912023-09-22T23:30:43.634Z 6 120-139 110 110 110 167 167 167 F F F
146922023-09-22T23:30:43.638Z 7 140-159 108 108 108 165 165 165 F F F
146932023-09-22T23:30:43.641Z 8 160-179 107 107 107 164 164 164 F F F
146942023-09-22T23:30:43.644Z 9 180-199 111 111 111 168 168 168 F F F
146952023-09-22T23:30:43.647Z 10 200-219 105 105 105 160 160 160 F F F
146962023-09-22T23:30:43.650Z 11 220-239 107 107 107 164 164 164 F F F
146972023-09-22T23:30:43.653Z 12 240-259 110 110 110 167 167 167 F F F
146982023-09-22T23:30:43.656Z 13 260-279 102 102 102 157 157 157 F F F
146992023-09-22T23:30:43.659Z 14 280-299 110 110 110 166 166 166 F F F
147002023-09-22T23:30:43.662Z 15 300-319 110 110 110 166 166 166 F F F
147012023-09-22T23:30:43.666Z 16 320-339 105 105 105 160 160 160 F F F
147022023-09-22T23:30:43.669Z 17 340-359 110 110 110 166 166 166 F F F
147032023-09-22T23:30:43.672Z 18 360-379 110 110 110 166 166 166 F F F
147042023-09-22T23:30:43.675Z 19 380-399 108 108 108 165 165 165 F F F
147052023-09-22T23:30:43.678Z 20 400-419 110 110 110 167 167 167 F F F
147062023-09-22T23:30:43.681Z 21 420-439 110 110 110 166 166 166 F F F
147072023-09-22T23:30:43.684Z 22 440-459 108 108 108 165 165 165 F F F
147082023-09-22T23:30:43.687Z 23 460-479 106 106 106 163 163 163 F F F
147092023-09-22T23:30:43.692Z 24 480-499 108 108 108 165 165 165 F F F
147102023-09-22T23:30:43.695Z 25 500-519 105 105 105 160 160 160 F F F
147112023-09-22T23:30:43.698Z 26 520-539 108 108 108 165 165 165 F F F
147122023-09-22T23:30:43.701Z 27 540-559 108 108 108 165 165 165 F F F
147132023-09-22T23:30:43.704Z 28 560-579 110 110 110 166 166 166 F F F
147142023-09-22T23:30:43.707Z 29 580-599 108 108 108 165 165 165 F F F
147152023-09-22T23:30:43.710ZMax gen: 111, Max flush: 168
147162023-09-22T23:30:43.713Z
147172023-09-22T23:30:43.716ZBegin loop 56
147182023-09-22T23:30:43.719ZDownstairs to restart: 2
147192023-09-22T23:30:43.722Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.627013589Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147202023-09-22T23:30:43.725Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.627379202Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147212023-09-22T23:30:43.728Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.627430717Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147222023-09-22T23:30:43.731Z{"msg":"Crucible stats registered with UUID: 988f8ceb-8072-4a11-8c9c-8fcf261ad1ee","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.627482403Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147232023-09-22T23:30:43.734Z{"msg":"Crucible 988f8ceb-8072-4a11-8c9c-8fcf261ad1ee has session id: 8fab0844-2bc9-4395-a81f-c5da470c7f1e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.627546069Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147242023-09-22T23:30:43.737ZCrucible runtime is spawned
147252023-09-22T23:30:43.741Z{"msg":"up_listen starts","v":The guest has requested activation with gen:112
147262023-09-22T23:30:43.744Z0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.627789494Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625,"task":"up_listen"}
147272023-09-22T23:30:43.747Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.627868176Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147282023-09-22T23:30:43.752Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.627931202Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147292023-09-22T23:30:43.755Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.627994779Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625,"looper":"1"}
147302023-09-22T23:30:43.758Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.628287054Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625,"looper":"0"}
147312023-09-22T23:30:43.761Z{"msg":"[1] 988f8ceb-8072-4a11-8c9c-8fcf261ad1ee looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.628337572Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625,"looper":"1"}
147322023-09-22T23:30:43.764Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.628450147Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625,"looper":"2"}
147332023-09-22T23:30:43.767Z{"msg":"[0] 988f8ceb-8072-4a11-8c9c-8fcf261ad1ee looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.628507655Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625,"looper":"0"}
147342023-09-22T23:30:43.770Z{"msg":"988f8ceb-8072-4a11-8c9c-8fcf261ad1ee active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.628908677Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147352023-09-22T23:30:43.773Z{"msg":"Set desired generation to :112","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.628960837Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147362023-09-22T23:30:43.775Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.629076438Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147372023-09-22T23:30:43.779Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.629465529Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147382023-09-22T23:30:43.782Z{"msg":"[1] 988f8ceb-8072-4a11-8c9c-8fcf261ad1ee (29057db3-92e8-4ba0-8faa-2c25208ffe7f) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.629734939Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147392023-09-22T23:30:43.785Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.629866783Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147402023-09-22T23:30:43.788Z{"msg":"[1] client is_active_req TRUE, promote! session 29057db3-92e8-4ba0-8faa-2c25208ffe7f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.629921457Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147412023-09-22T23:30:43.793Z{"msg":"[0] 988f8ceb-8072-4a11-8c9c-8fcf261ad1ee (29057db3-92e8-4ba0-8faa-2c25208ffe7f) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.62998268Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147422023-09-22T23:30:43.796Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.630053594Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147432023-09-22T23:30:43.799Z{"msg":"[0] client is_active_req TRUE, promote! session 29057db3-92e8-4ba0-8faa-2c25208ffe7f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.630145323Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147442023-09-22T23:30:43.802Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.630513401Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147452023-09-22T23:30:43.805Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.630569774Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147462023-09-22T23:30:43.808Z{"msg":"988f8ceb-8072-4a11-8c9c-8fcf261ad1ee WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.630699455Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147472023-09-22T23:30:43.811Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.630831401Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147482023-09-22T23:30:43.814Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.630894489Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147492023-09-22T23:30:43.817Z{"msg":"988f8ceb-8072-4a11-8c9c-8fcf261ad1ee WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.630936902Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147502023-09-22T23:30:43.820Z{"msg":"[1] 988f8ceb-8072-4a11-8c9c-8fcf261ad1ee (29057db3-92e8-4ba0-8faa-2c25208ffe7f) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.633376292Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147512023-09-22T23:30:43.823Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.633485526Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147522023-09-22T23:30:43.826Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:30:43.633547325Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147532023-09-22T23:30:43.829Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.633607021Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147542023-09-22T23:30:43.832Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.633662061Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147552023-09-22T23:30:43.835Z{"msg":"988f8ceb-8072-4a11-8c9c-8fcf261ad1ee WaitActive WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.633716145Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147562023-09-22T23:30:43.839Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.633770318Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147572023-09-22T23:30:43.842Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.633799927Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147582023-09-22T23:30:43.845Z{"msg":"[0] 988f8ceb-8072-4a11-8c9c-8fcf261ad1ee (29057db3-92e8-4ba0-8faa-2c25208ffe7f) WaitActive WaitQuorum New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.633821941Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147592023-09-22T23:30:43.848Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.633840183Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147602023-09-22T23:30:43.851Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:30:43.633857503Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147612023-09-22T23:30:43.854Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.633874967Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147622023-09-22T23:30:43.857Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.633892907Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147632023-09-22T23:30:43.860Z{"msg":"988f8ceb-8072-4a11-8c9c-8fcf261ad1ee WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.633910953Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147642023-09-22T23:30:43.863Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.63392825Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147652023-09-22T23:30:43.867Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:43.633946112Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147662023-09-22T23:30:44.629Z{"msg":"[2] 988f8ceb-8072-4a11-8c9c-8fcf261ad1ee looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.629932742Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625,"looper":"2"}
147672023-09-22T23:30:44.633Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.630024012Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147682023-09-22T23:30:44.635Z{"msg":"[2] 988f8ceb-8072-4a11-8c9c-8fcf261ad1ee (29057db3-92e8-4ba0-8faa-2c25208ffe7f) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.63115847Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147692023-09-22T23:30:44.638Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.631214234Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147702023-09-22T23:30:44.642Z{"msg":"[2] client is_active_req TRUE, promote! session 29057db3-92e8-4ba0-8faa-2c25208ffe7f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.631237972Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147712023-09-22T23:30:44.645Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.631870139Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147722023-09-22T23:30:44.648Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.631926646Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147732023-09-22T23:30:44.651Z{"msg":"988f8ceb-8072-4a11-8c9c-8fcf261ad1ee WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.631968736Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147742023-09-22T23:30:44.654Z{"msg":"[2] 988f8ceb-8072-4a11-8c9c-8fcf261ad1ee (29057db3-92e8-4ba0-8faa-2c25208ffe7f) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.634335595Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147752023-09-22T23:30:44.658Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.634390284Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147762023-09-22T23:30:44.661Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:30:44.634412202Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147772023-09-22T23:30:44.664Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.634450973Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147782023-09-22T23:30:44.667Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.634490253Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147792023-09-22T23:30:44.670Z{"msg":"988f8ceb-8072-4a11-8c9c-8fcf261ad1ee WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.634513622Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147802023-09-22T23:30:44.673Z{"msg":"[0]R flush_numbers[0..12]: [162, 168, 165, 163, 163, 167, 167, 165, 164, 168, 160, 164]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.634552118Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147812023-09-22T23:30:44.677Z{"msg":"[0]R generation[0..12]: [106, 111, 108, 106, 106, 110, 110, 108, 107, 111, 105, 107]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.634591781Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147822023-09-22T23:30:44.680Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.634632439Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147832023-09-22T23:30:44.684Z{"msg":"[1]R flush_numbers[0..12]: [162, 168, 165, 163, 163, 167, 167, 165, 164, 168, 160, 164]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.634681974Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147842023-09-22T23:30:44.687Z{"msg":"[1]R generation[0..12]: [106, 111, 108, 106, 106, 110, 110, 108, 107, 111, 105, 107]","v":0,"name":"crucible","level":30,"time":"The guest has finished waiting for activation with:112
147852023-09-22T23:30:44.690ZWait for a query_work_queue command to finish before sending IO
147862023-09-22T23:30:44.693Z2023-09-22T23:30:44.634721373Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147872023-09-22T23:30:44.696Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.634959972Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147882023-09-22T23:30:44.699Z{"msg":"[2]R flush_numbers[0..12]: [162, 168, 165, 163, 163, 167, 167, 165, 164, 168, 160, 164]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.635027181Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147892023-09-22T23:30:44.702Z{"msg":"[2]R generation[0..12]: [106, 111, 108, 106, 106, 110, 110, 108, 107, 111, 105, 107]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.6350891Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147902023-09-22T23:30:44.705Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.635207274Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147912023-09-22T23:30:44.708Z{"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.635251312Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147922023-09-22T23:30:44.712Z{"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.635271979Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147932023-09-22T23:30:44.715Z{"msg":"Next flush: 169","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.635308193Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147942023-09-22T23:30:44.719Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.63535092Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147952023-09-22T23:30:44.722Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.635399215Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147962023-09-22T23:30:44.725Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.63543712Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147972023-09-22T23:30:44.728Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.635478775Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147982023-09-22T23:30:44.731Z{"msg":"988f8ceb-8072-4a11-8c9c-8fcf261ad1ee is now active with session: 29057db3-92e8-4ba0-8faa-2c25208ffe7f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.635518291Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
147992023-09-22T23:30:44.734Z{"msg":"988f8ceb-8072-4a11-8c9c-8fcf261ad1ee Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.635557101Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
148002023-09-22T23:30:44.737Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.635595621Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
148012023-09-22T23:30:44.740Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.635637951Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
148022023-09-22T23:30:44.743Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.635675638Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
148032023-09-22T23:30:44.747Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.63571563Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
148042023-09-22T23:30:44.749Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.635753743Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
148052023-09-22T23:30:44.753Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.635798053Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
148062023-09-22T23:30:44.756Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.635838249Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
148072023-09-22T23:30:44.759Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.635875376Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
148082023-09-22T23:30:44.762Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.635912751Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
148092023-09-22T23:30:44.765Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
148102023-09-22T23:30:44.768Z,"time":"2023-09-22T23:30:44.63595054Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
148112023-09-22T23:30:44.771Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.635988168Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1625}
148122023-09-22T23:30:44.774ZLoading write count information from file "/tmp/test_repair_verify.data"
148132023-09-22T23:30:44.777ZRun Repair workload
148142023-09-22T23:30:44.780Z01/30 Write block 167 len 16384 data: 6 5 5 9
148152023-09-22T23:30:44.783Z02/30 Read block 255 len 8192
148162023-09-22T23:30:44.786Z03/30 Read block 120 len 8192
148172023-09-22T23:30:44.789Z04/30 Read block 408 len 16384
148182023-09-22T23:30:44.792Z05/30 Read block 80 len 8192
148192023-09-22T23:30:44.795Z06/30 Read block 383 len 28672
148202023-09-22T23:30:44.799Z07/30 Read block 142 len 36864
148212023-09-22T23:30:44.802Z08/30 Flush
148222023-09-22T23:30:44.805Z09/30 Flush
148232023-09-22T23:30:44.808Z10/30 Write block 540 len 24576 data: 9 6 7 8 7 6
148242023-09-22T23:30:44.812Z11/30 Write block 98 len 32768 data: 12 10 9 9 10 11 13 15
148252023-09-22T23:30:44.815Z12/30 Write block 577 len 8192 data: 9 8
148262023-09-22T23:30:44.818Z13/30 Read block 540 len 12288
148272023-09-22T23:30:44.821Z14/30 Read block 246 len 8192
148282023-09-22T23:30:44.824Z15/30 Read block 327 len 24576
148292023-09-22T23:30:44.827Z16/30 Read block 522 len 40960
148302023-09-22T23:30:44.830Z17/30 Write block 506 len 8192 data: 9 9
148312023-09-22T23:30:44.833Z18/30 Read block 251 len 4096
148322023-09-22T23:30:44.838Z19/30 Read block 334 len 40960
148332023-09-22T23:30:44.841Z20/30 Read block 335 len 16384
148342023-09-22T23:30:44.844Z21/30 Write block 565 len 40960 data: 10 11 13 14 14 10 8 5 4 4
148352023-09-22T23:30:44.847Z22/30 Write block 560 len 16384 data: 9 7 6 7
148362023-09-22T23:30:44.850Z23/30 Read block 8 len 16384
148372023-09-22T23:30:44.853Z24/30 Read block 579 len 4096
148382023-09-22T23:30:44.856Z25/30 Read block 69 len 20480
148392023-09-22T23:30:44.859Z26/30 Write block 349 len 28672 data: 7 6 8 8 7 8 9
148402023-09-22T23:30:44.862Z27/30 Write block 261 len 28672 data: 11 12 12 12 11 11 14
148412023-09-22T23:30:44.866Z28/30 Read block 502 len 36864
148422023-09-22T23:30:44.868Z29/30 Write block 560 len 24576 data: 10 8 7 8 10 11
148432023-09-22T23:30:44.871Z30/30 Write block 69 len 36864 data: 16 9 8 8 5 6 8 8 10
148442023-09-22T23:30:44.874Z----------------------------------------------------------------
148452023-09-22T23:30:44.877Z Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30
148462023-09-22T23:30:44.880ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
148472023-09-22T23:30:44.883Z 1 Acked 1000 Write 4 Done Done Done false
148482023-09-22T23:30:44.886Z 2 Acked 1001 Read 2 Done Done Done false
148492023-09-22T23:30:44.889Z 3 Acked 1002 Read 2 Done Done Done false
148502023-09-22T23:30:44.892Z 4 Acked 1003 Read 4 Done Done Done false
148512023-09-22T23:30:44.895Z 5 Acked 1004 Read 2 Done Done Done false
148522023-09-22T23:30:44.898Z 6 Acked 1005 Read 7 Done Done Done false
148532023-09-22T23:30:44.901Z 7 Acked 1006 Read 9 Done Done Done false
148542023-09-22T23:30:44.904Z 8 Acked 1007 Flush 0 Done Done Sent false
148552023-09-22T23:30:44.908Z 9 Acked 1008 Flush 0 Done Done Sent false
148562023-09-22T23:30:44.911Z 10 Acked 1009 Write 6 Done Done Sent false
148572023-09-22T23:30:44.914Z 11 Acked 1010 Write 8 Done Done Sent false
148582023-09-22T23:30:44.918Z 12 Acked 1011 Write 2 Done Done Sent false
148592023-09-22T23:30:44.922Z 13 Acked 1012 Read 3 Done Done Sent false
148602023-09-22T23:30:44.925Z 14 Acked 1013 Read 2 Done Done Sent false
148612023-09-22T23:30:44.929Z 15 Acked 1014 Read 6 Done Done Sent false
148622023-09-22T23:30:44.931Z 16 Acked 1015 Read 10 Done Done Sent false
148632023-09-22T23:30:44.934Z 17 Acked 1016 Write 2 Done Done Sent false
148642023-09-22T23:30:44.937Z 18 Acked 1017 Read 1 Done Done Sent false
148652023-09-22T23:30:44.940Z 19 Acked 1018 Read 10 Done Done Sent false
148662023-09-22T23:30:44.943Z 20 Acked 1019 Read 4 Done Done Sent false
148672023-09-22T23:30:44.946Z 21 Acked 1020 Write 10 Done Done Sent false
148682023-09-22T23:30:44.949Z 22 Acked 1021 Write 4 Done Done Sent false
148692023-09-22T23:30:44.952Z 23 Acked 1022 Read 4 Done Done Sent false
148702023-09-22T23:30:44.955Z 24 Acked 1023 Read 1 Done Done Sent false
148712023-09-22T23:30:44.958Z 25 Acked 1024 Read 5 Done Done Sent false
148722023-09-22T23:30:44.961Z 26 Acked 1025 Write 7 Done Done Sent false
148732023-09-22T23:30:44.964Z 27 Acked 1026 Write 7 Done Done Sent false
148742023-09-22T23:30:44.967Z 28 Acked 1027 Read 9 Done Done Sent false
148752023-09-22T23:30:44.970Z 29 Acked 1028 Write 6 Sent Sent Sent false
148762023-09-22T23:30:44.973Z 30 Acked 1029 Write 9 Sent Sent Sent false
148772023-09-22T23:30:44.976Z STATES DS:0 DS:1 DS:2 TOTAL
148782023-09-22T23:30:44.978Z New 0 0 0 0
148792023-09-22T23:30:44.981Z Sent 2 2 23 27
148802023-09-22T23:30:44.984Z Done 28 28 7 63
148812023-09-22T23:30:44.987Z Skipped 0 0 0 0
148822023-09-22T23:30:44.990Z Error 0 0 0 0
148832023-09-22T23:30:44.994ZLast Flush: 1008 1008 0
148842023-09-22T23:30:45.403ZDownstairs last five completed:
148852023-09-22T23:30:45.414ZUpstairs last five completed: 30 29 28 27 26 25
148862023-09-22T23:30:45.417ZWrote out file "/tmp/test_repair_verify.data"
148872023-09-22T23:30:45.421Z
148882023-09-22T23:30:45.425ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
148892023-09-22T23:30:45.428Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.759551798Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1626}
148902023-09-22T23:30:45.431Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.759863584Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1626}
148912023-09-22T23:30:45.434Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.759887236Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1626}
148922023-09-22T23:30:45.437Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.789840363Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1626}
148932023-09-22T23:30:45.440Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.789891494Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1626}
148942023-09-22T23:30:45.443Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.789922754Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1626}
148952023-09-22T23:30:45.446Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.813250243Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1626}
148962023-09-22T23:30:45.449Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.813283819Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1626}
148972023-09-22T23:30:45.452ZError: Difference in extent metadata found!
148982023-09-22T23:30:45.455Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:44.813303087Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1626}
148992023-09-22T23:30:45.458ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
149002023-09-22T23:30:45.465Z 0 000-019 106 106 106 162 162 162 F F F
149012023-09-22T23:30:45.468Z 1 020-039 111 111 111 168 168 168 F F F
149022023-09-22T23:30:45.471Z 2 040-059 108 108 108 165 165 165 F F F
149032023-09-22T23:30:45.474Z 3 060-079 106 106 106 163 163 163 F T F <---
149042023-09-22T23:30:45.477Z 4 080-099 106 106 106 163 163 163 T T F <---
149052023-09-22T23:30:45.480Z 5 100-119 110 110 110 167 167 167 T T F <---
149062023-09-22T23:30:45.484Z 6 120-139 110 110 110 167 167 167 F F F
149072023-09-22T23:30:45.487Z 7 140-159 108 108 108 165 165 165 F F F
149082023-09-22T23:30:45.491Z 8 160-179 112 112 107 169 169 164 F F T <---
149092023-09-22T23:30:45.495Z 9 180-199 111 111 111 168 168 168 F F F
149102023-09-22T23:30:45.499Z 10 200-219 105 105 105 160 160 160 F F F
149112023-09-22T23:30:45.502Z 11 220-239 107 107 107 164 164 164 F F F
149122023-09-22T23:30:45.505Z 12 240-259 110 110 110 167 167 167 F F F
149132023-09-22T23:30:45.508Z 13 260-279 102 102 102 157 157 157 T T F <---
149142023-09-22T23:30:45.511Z 14 280-299 110 110 110 166 166 166 F F F
149152023-09-22T23:30:45.514Z 15 300-319 110 110 110 166 166 166 F F F
149162023-09-22T23:30:45.518Z 16 320-339 105 105 105 160 160 160 F F F
149172023-09-22T23:30:45.522Z 17 340-359 110 110 110 166 166 166 T T F <---
149182023-09-22T23:30:45.525Z 18 360-379 110 110 110 166 166 166 F F F
149192023-09-22T23:30:45.528Z 19 380-399 108 108 108 165 165 165 F F F
149202023-09-22T23:30:45.532Z 20 400-419 110 110 110 167 167 167 F F F
149212023-09-22T23:30:45.535Z 21 420-439 110 110 110 166 166 166 F F F
149222023-09-22T23:30:45.538Z 22 440-459 108 108 108 165 165 165 F F F
149232023-09-22T23:30:45.542Z 23 460-479 106 106 106 163 163 163 F F F
149242023-09-22T23:30:45.545Z 24 480-499 108 108 108 165 165 165 F F F
149252023-09-22T23:30:45.548Z 25 500-519 105 105 105 160 160 160 T T F <---
149262023-09-22T23:30:45.551Z 26 520-539 108 108 108 165 165 165 F F F
149272023-09-22T23:30:45.555Z 27 540-559 108 108 108 165 165 165 T T F <---
149282023-09-22T23:30:45.558Z 28 560-579 110 110 110 166 166 166 T T F <---
149292023-09-22T23:30:45.561Z 29 580-599 108 108 108 165 165 165 F F F
149302023-09-22T23:30:45.564ZMax gen: 112, Max flush: 169
149312023-09-22T23:30:45.574ZA Difference in extent metadata is expected here
149322023-09-22T23:30:45.577ZOn loop 56
149332023-09-22T23:30:45.582Z
149342023-09-22T23:30:45.585ZVerifying data now
149352023-09-22T23:30:50.925ZNo change after verify
149362023-09-22T23:30:50.928ZLoop: 56 Downstairs dump after verify (and repair):
149372023-09-22T23:30:50.944Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:50.943885826Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1631}
149382023-09-22T23:30:50.948Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:50.944293639Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1631}
149392023-09-22T23:30:50.951Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:50.944319321Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1631}
149402023-09-22T23:30:50.964Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:50.963180884Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1631}
149412023-09-22T23:30:50.967Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:50.963234259Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1631}
149422023-09-22T23:30:50.971Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:50.96325456Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1631}
149432023-09-22T23:30:50.974Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:50.974237214Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1631}
149442023-09-22T23:30:50.978Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:50.974289089Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1631}
149452023-09-22T23:30:50.981Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:50.974327227Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1631}
149462023-09-22T23:30:50.985ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
149472023-09-22T23:30:50.989Z 0 000-019 106 106 106 162 162 162 F F F
149482023-09-22T23:30:50.992Z 1 020-039 111 111 111 168 168 168 F F F
149492023-09-22T23:30:50.996Z 2 040-059 108 108 108 165 165 165 F F F
149502023-09-22T23:30:51.000Z 3 060-079 113 113 113 170 170 170 F F F
149512023-09-22T23:30:51.004Z 4 080-099 113 113 113 170 170 170 F F F
149522023-09-22T23:30:51.008Z 5 100-119 113 113 113 170 170 170 F F F
149532023-09-22T23:30:51.011Z 6 120-139 110 110 110 167 167 167 F F F
149542023-09-22T23:30:51.015Z 7 140-159 108 108 108 165 165 165 F F F
149552023-09-22T23:30:51.019Z 8 160-179 112 112 112 169 169 169 F F F
149562023-09-22T23:30:51.022Z 9 180-199 111 111 111 168 168 168 F F F
149572023-09-22T23:30:51.026Z 10 200-219 105 105 105 160 160 160 F F F
149582023-09-22T23:30:51.030Z 11 220-239 107 107 107 164 164 164 F F F
149592023-09-22T23:30:51.033Z 12 240-259 110 110 110 167 167 167 F F F
149602023-09-22T23:30:51.037Z 13 260-279 113 113 113 170 170 170 F F F
149612023-09-22T23:30:51.040Z 14 280-299 110 110 110 166 166 166 F F F
149622023-09-22T23:30:51.043Z 15 300-319 110 110 110 166 166 166 F F F
149632023-09-22T23:30:51.046Z 16 320-339 105 105 105 160 160 160 F F F
149642023-09-22T23:30:51.050Z 17 340-359 113 113 113 170 170 170 F F F
149652023-09-22T23:30:51.054Z 18 360-379 110 110 110 166 166 166 F F F
149662023-09-22T23:30:51.057Z 19 380-399 108 108 108 165 165 165 F F F
149672023-09-22T23:30:51.060Z 20 400-419 110 110 110 167 167 167 F F F
149682023-09-22T23:30:51.064Z 21 420-439 110 110 110 166 166 166 F F F
149692023-09-22T23:30:51.067Z 22 440-459 108 108 108 165 165 165 F F F
149702023-09-22T23:30:51.071Z 23 460-479 106 106 106 163 163 163 F F F
149712023-09-22T23:30:51.074Z 24 480-499 108 108 108 165 165 165 F F F
149722023-09-22T23:30:51.078Z 25 500-519 113 113 113 170 170 170 F F F
149732023-09-22T23:30:51.082Z 26 520-539 108 108 108 165 165 165 F F F
149742023-09-22T23:30:51.085Z 27 540-559 113 113 113 170 170 170 F F F
149752023-09-22T23:30:51.089Z 28 560-579 113 113 113 170 170 170 F F F
149762023-09-22T23:30:51.092Z 29 580-599 108 108 108 165 165 165 F F F
149772023-09-22T23:30:51.095ZMax gen: 113, Max flush: 170
149782023-09-22T23:30:51.099Z
149792023-09-22T23:30:51.104ZBegin loop 57
149802023-09-22T23:30:51.109ZDownstairs to restart: 2
149812023-09-22T23:30:51.112Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.023177348Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
149822023-09-22T23:30:51.116Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.023605754Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
149832023-09-22T23:30:51.120Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.023678003Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
149842023-09-22T23:30:51.124Z{"msg":"Crucible stats registered with UUID: 4bea6a81-2fc9-4bb6-8d0c-ea2e33dcfb60","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.023749106Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
149852023-09-22T23:30:51.128ZCrucible runtime is spawned
149862023-09-22T23:30:51.131Z{"msg":"Crucible 4bea6a81-2fc9-4bb6-8d0c-ea2e33dcfb60 has session id: d1ea70c4-6c44-42a8-bdea-ef5fad26ab0e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.023893161Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
149872023-09-22T23:30:51.134ZThe guest has requested activation with gen:114
149882023-09-22T23:30:51.138Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.023973432Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633,"task":"up_listen"}
149892023-09-22T23:30:51.142Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.024040223Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
149902023-09-22T23:30:51.145Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.024085253Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
149912023-09-22T23:30:51.148Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.024124687Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633,"looper":"0"}
149922023-09-22T23:30:51.152Z{"msg":"[0] 4bea6a81-2fc9-4bb6-8d0c-ea2e33dcfb60 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.024348448Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633,"looper":"0"}
149932023-09-22T23:30:51.155Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.024427712Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633,"looper":"1"}
149942023-09-22T23:30:51.159Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.024760504Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633,"looper":"2"}
149952023-09-22T23:30:51.162Z{"msg":"[1] 4bea6a81-2fc9-4bb6-8d0c-ea2e33dcfb60 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.024846759Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633,"looper":"1"}
149962023-09-22T23:30:51.165Z{"msg":"4bea6a81-2fc9-4bb6-8d0c-ea2e33dcfb60 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.025187932Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
149972023-09-22T23:30:51.169Z{"msg":"Set desired generation to :114","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.025253303Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
149982023-09-22T23:30:51.172Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.02531198Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
149992023-09-22T23:30:51.175Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.025646749Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150002023-09-22T23:30:51.179Z{"msg":"[0] 4bea6a81-2fc9-4bb6-8d0c-ea2e33dcfb60 (41a76128-6822-4071-811d-409afea18cbe) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.026093409Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150012023-09-22T23:30:51.183Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.026151455Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150022023-09-22T23:30:51.186Z{"msg":"[0] client is_active_req TRUE, promote! session 41a76128-6822-4071-811d-409afea18cbe","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.026194611Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150032023-09-22T23:30:51.190Z{"msg":"[1] 4bea6a81-2fc9-4bb6-8d0c-ea2e33dcfb60 (41a76128-6822-4071-811d-409afea18cbe) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.026275599Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150042023-09-22T23:30:51.193Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.026378615Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150052023-09-22T23:30:51.196Z{"msg":"[1] client is_active_req TRUE, promote! session 41a76128-6822-4071-811d-409afea18cbe","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.026461361Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150062023-09-22T23:30:51.200Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.026836074Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150072023-09-22T23:30:51.207Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.02690007Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150082023-09-22T23:30:51.211Z{"msg":"4bea6a81-2fc9-4bb6-8d0c-ea2e33dcfb60 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.027016429Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150092023-09-22T23:30:51.215Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.027148738Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150102023-09-22T23:30:51.218Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.027218408Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150112023-09-22T23:30:51.222Z{"msg":"4bea6a81-2fc9-4bb6-8d0c-ea2e33dcfb60 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.027258067Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150122023-09-22T23:30:51.225Z{"msg":"[0] 4bea6a81-2fc9-4bb6-8d0c-ea2e33dcfb60 (41a76128-6822-4071-811d-409afea18cbe) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.029096171Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150132023-09-22T23:30:51.229Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.02918347Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150142023-09-22T23:30:51.233Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:30:51.02922175Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150152023-09-22T23:30:51.236Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.029293551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150162023-09-22T23:30:51.240Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.029366339Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150172023-09-22T23:30:51.244Z{"msg":"4bea6a81-2fc9-4bb6-8d0c-ea2e33dcfb60 WaitQuorum WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.029438512Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150182023-09-22T23:30:51.247Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.029512437Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150192023-09-22T23:30:51.251Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.029582158Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150202023-09-22T23:30:51.254Z{"msg":"[1] 4bea6a81-2fc9-4bb6-8d0c-ea2e33dcfb60 (41a76128-6822-4071-811d-409afea18cbe) WaitQuorum WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.029659717Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150212023-09-22T23:30:51.258Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.029735001Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150222023-09-22T23:30:51.262Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:30:51.029789406Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150232023-09-22T23:30:51.265Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.029830088Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150242023-09-22T23:30:51.268Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.029850589Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150252023-09-22T23:30:51.272Z{"msg":"4bea6a81-2fc9-4bb6-8d0c-ea2e33dcfb60 WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.029868762Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150262023-09-22T23:30:51.275Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.029886288Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150272023-09-22T23:30:51.279Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:51.029903329Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150282023-09-22T23:30:52.027Z{"msg":"[2] 4bea6a81-2fc9-4bb6-8d0c-ea2e33dcfb60 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.026730915Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633,"looper":"2"}
150292023-09-22T23:30:52.031Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.026820095Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150302023-09-22T23:30:52.035Z{"msg":"[2] 4bea6a81-2fc9-4bb6-8d0c-ea2e33dcfb60 (41a76128-6822-4071-811d-409afea18cbe) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.027880676Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150312023-09-22T23:30:52.038Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.027938117Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150322023-09-22T23:30:52.042Z{"msg":"[2] client is_active_req TRUE, promote! session 41a76128-6822-4071-811d-409afea18cbe","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.027982229Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150332023-09-22T23:30:52.046Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.02857274Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150342023-09-22T23:30:52.049Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.028633426Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150352023-09-22T23:30:52.053Z{"msg":"4bea6a81-2fc9-4bb6-8d0c-ea2e33dcfb60 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.028677681Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150362023-09-22T23:30:52.056Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.031052581Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150372023-09-22T23:30:52.059Z{"msg":"[2] 4bea6a81-2fc9-4bb6-8d0c-ea2e33dcfb60 (41a76128-6822-4071-811d-409afea18cbe) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.031222529Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150382023-09-22T23:30:52.063Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.031289151Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150392023-09-22T23:30:52.067Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:30:52.031337555Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150402023-09-22T23:30:52.071Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.031377822Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150412023-09-22T23:30:52.075Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.031416703Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150422023-09-22T23:30:52.078Z{"msg":"4bea6a81-2fc9-4bb6-8d0c-ea2e33dcfb60 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.031456275Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150432023-09-22T23:30:52.082Z{"msg":"[0]R flush_numbers[0..12]: [162, 168, 165, 170, 170, 170, 167, 165, 169, 168, 160, 164]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.031496213Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150442023-09-22T23:30:52.086Z{"msg":"[0]R generation[0..12]: [106, 111, 108, 113, 113, 113, 110, 108, 112, 111, 105, 107]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.031536789Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150452023-09-22T23:30:52.090Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.031585185Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150462023-09-22T23:30:52.093Z{"msg":"[1]R flush_numbers[0..12]: [162, 168, 165, 170, 170, 170, 167, 165, 169, 168, 160, 164]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.031625143Z","hostname":"The guest has finished waiting for activation with:114
150472023-09-22T23:30:52.097ZWait for a query_work_queue command to finish before sending IO
150482023-09-22T23:30:52.100Zip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150492023-09-22T23:30:52.104Z{"msg":"[1]R generation[0..12]: [106, 111, 108, 113, 113, 113, 110, 108, 112, 111, 105, 107]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.031806083Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150502023-09-22T23:30:52.108Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.031884129Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150512023-09-22T23:30:52.112Z{"msg":"[2]R flush_numbers[0..12]: [162, 168, 165, 170, 170, 170, 167, 165, 169, 168, 160, 164]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.031932796Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150522023-09-22T23:30:52.115Z{"msg":"[2]R generation[0..12]: [106, 111, 108, 113, 113, 113, 110, 108, 112, 111, 105, 107]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.031973792Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150532023-09-22T23:30:52.119Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.031997133Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150542023-09-22T23:30:52.123Z{"msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.032041865Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150552023-09-22T23:30:52.127Z{"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.032141656Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150562023-09-22T23:30:52.130Z{"msg":"Next flush: 171","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.032183312Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150572023-09-22T23:30:52.134Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.03223846Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150582023-09-22T23:30:52.138Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.032284528Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150592023-09-22T23:30:52.142Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.032327252Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150602023-09-22T23:30:52.145Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.032365794Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150612023-09-22T23:30:52.149Z{"msg":"4bea6a81-2fc9-4bb6-8d0c-ea2e33dcfb60 is now active with session: 41a76128-6822-4071-811d-409afea18cbe","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.032412731Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150622023-09-22T23:30:52.152Z{"msg":"4bea6a81-2fc9-4bb6-8d0c-ea2e33dcfb60 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.032462641Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150632023-09-22T23:30:52.156Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.032502275Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150642023-09-22T23:30:52.159Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.032540491Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150652023-09-22T23:30:52.163Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.032588541Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150662023-09-22T23:30:52.166Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.03263507Z",Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
150672023-09-22T23:30:52.170Z"hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150682023-09-22T23:30:52.174Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.032738538Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150692023-09-22T23:30:52.177Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.032783414Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150702023-09-22T23:30:52.181Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.032821896Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150712023-09-22T23:30:52.184Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.032859181Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150722023-09-22T23:30:52.188Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.032896539Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150732023-09-22T23:30:52.191Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.032933562Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150742023-09-22T23:30:52.194Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.032970146Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1633}
150752023-09-22T23:30:52.198ZLoading write count information from file "/tmp/test_repair_verify.data"
150762023-09-22T23:30:52.201ZRun Repair workload
150772023-09-22T23:30:52.205Z01/30 Write block 252 len 24576 data: 9 9 10 9 8 8
150782023-09-22T23:30:52.208Z02/30 Read block 59 len 24576
150792023-09-22T23:30:52.212Z03/30 Read block 49 len 16384
150802023-09-22T23:30:52.216Z04/30 Write block 176 len 36864 data: 14 13 13 10 8 9 11 8 10
150812023-09-22T23:30:52.220Z05/30 Write block 463 len 16384 data: 9 10 14 11
150822023-09-22T23:30:52.224Z06/30 Write block 176 len 12288 data: 15 14 14
150832023-09-22T23:30:52.227Z07/30 Read block 425 len 12288
150842023-09-22T23:30:52.229Z08/30 Read block 231 len 24576
150852023-09-22T23:30:52.233Z09/30 Read block 460 len 12288
150862023-09-22T23:30:52.236Z10/30 Write block 499 len 32768 data: 9 9 8 9 8 9 8 10
150872023-09-22T23:30:52.239Z11/30 Read block 132 len 24576
150882023-09-22T23:30:52.242Z12/30 Write block 433 len 36864 data: 12 10 11 10 10 9 7 7 10
150892023-09-22T23:30:52.248Z13/30 Read block 486 len 20480
150902023-09-22T23:30:52.251Z14/30 Read block 211 len 8192
150912023-09-22T23:30:52.254Z15/30 Read block 595 len 16384
150922023-09-22T23:30:52.257Z16/30 Read block 389 len 24576
150932023-09-22T23:30:52.260Z17/30 Write block 199 len 28672 data: 6 8 8 10 11 10 15
150942023-09-22T23:30:52.264Z18/30 Read block 20 len 36864
150952023-09-22T23:30:52.266Z19/30 Write block 214 len 32768 data: 11 11 10 10 8 6 8 7
150962023-09-22T23:30:52.270Z20/30 Write block 331 len 16384 data: 9 9 9 11
150972023-09-22T23:30:52.273Z21/30 Read block 293 len 24576
150982023-09-22T23:30:52.276Z22/30 Flush
150992023-09-22T23:30:52.279Z23/30 Write block 559 len 32768 data: 8 11 9 8 9 11 12 12
151002023-09-22T23:30:52.282Z24/30 Write block 387 len 16384 data: 10 10 10 10
151012023-09-22T23:30:52.285Z25/30 Write block 128 len 4096 data: 8
151022023-09-22T23:30:52.288Z26/30 Read block 166 len 40960
151032023-09-22T23:30:52.292Z27/30 Read block 24 len 20480
151042023-09-22T23:30:52.295Z28/30 Write block 71 len 16384 data: 9 9 6 7
151052023-09-22T23:30:52.298Z29/30 Write block 549 len 36864 data: 13 13 13 13 12 7 7 8 7
151062023-09-22T23:30:52.301Z30/30 Write block 522 len 16384 data: 8 8 9 11
151072023-09-22T23:30:52.304Z----------------------------------------------------------------
151082023-09-22T23:30:52.308Z Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30
151092023-09-22T23:30:52.311ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
151102023-09-22T23:30:52.314Z 1 Acked 1000 Write 6 Done Done Done false
151112023-09-22T23:30:52.317Z 2 Acked 1001 Read 6 Done Done Sent false
151122023-09-22T23:30:52.322Z 3 Acked 1002 Read 4 Done Done Sent false
151132023-09-22T23:30:52.324Z 4 Acked 1003 Write 9 Done Done Sent false
151142023-09-22T23:30:52.327Z 5 Acked 1004 Write 4 Done Done Sent false
151152023-09-22T23:30:52.330Z 6 Acked 1005 Write 3 Done Done Sent false
151162023-09-22T23:30:52.333Z 7 Acked 1006 Read 3 Done Done Sent false
151172023-09-22T23:30:52.336Z 8 Acked 1007 Read 6 Done Done Sent false
151182023-09-22T23:30:52.339Z 9 Acked 1008 Read 3 Done Done Sent false
151192023-09-22T23:30:52.342Z 10 Acked 1009 Write 8 Done Done Sent false
151202023-09-22T23:30:52.345Z 11 Acked 1010 Read 6 Done Done Sent false
151212023-09-22T23:30:52.348Z 12 Acked 1011 Write 9 Done Done Sent false
151222023-09-22T23:30:52.351Z 13 Acked 1012 Read 5 Done Done Sent false
151232023-09-22T23:30:52.354Z 14 Acked 1013 Read 2 Done Done Sent false
151242023-09-22T23:30:52.357Z 15 Acked 1014 Read 4 Done Done Sent false
151252023-09-22T23:30:52.360Z 16 Acked 1015 Read 6 Done Done Sent false
151262023-09-22T23:30:52.364Z 17 Acked 1016 Write 7 Done Done Sent false
151272023-09-22T23:30:52.367Z 18 Acked 1017 Read 9 Done Done Sent false
151282023-09-22T23:30:52.370Z 19 Acked 1018 Write 8 Done Done Sent false
151292023-09-22T23:30:52.373Z 20 Acked 1019 Write 4 Done Done Sent false
151302023-09-22T23:30:52.376Z 21 Acked 1020 Read 6 Done Done Sent false
151312023-09-22T23:30:52.379Z 22 Acked 1021 Flush 0 Done Done Sent false
151322023-09-22T23:30:52.382Z 23 Acked 1022 Write 8 Done Done Sent false
151332023-09-22T23:30:52.385Z 24 Acked 1023 Write 4 Done Done Sent false
151342023-09-22T23:30:52.428Z 25 Acked 1024 Write 1 Done Done Sent false
151352023-09-22T23:30:52.432Z 26 Acked 1025 Read 10 Done Done Sent false
151362023-09-22T23:30:52.435Z 27 Acked 1026 Read 5 Done Done Sent false
151372023-09-22T23:30:52.439Z 28 Acked 1027 Write 4 Sent Sent Sent false
151382023-09-22T23:30:52.443Z 29 Acked 1028 Write 9 Sent Sent Sent false
151392023-09-22T23:30:52.447Z 30 Acked 1029 Write 4 Sent Sent Sent false
151402023-09-22T23:30:52.451Z STATES DS:0 DS:1 DS:2 TOTAL
151412023-09-22T23:30:52.457Z New 0 0 0 0
151422023-09-22T23:30:52.461Z Sent 3 3 29 35
151432023-09-22T23:30:52.465Z Done 27 27 1 55
151442023-09-22T23:30:52.470Z Skipped 0 0 0 0
151452023-09-22T23:30:52.475Z Error 0 0 0 0
151462023-09-22T23:30:52.479ZLast Flush: 1021 1021 0
151472023-09-22T23:30:52.483ZDownstairs last five completed:
151482023-09-22T23:30:52.487ZUpstairs last five completed: 30 29 28 27 26 25
151492023-09-22T23:30:52.491ZWrote out file "/tmp/test_repair_verify.data"
151502023-09-22T23:30:52.495Z
151512023-09-22T23:30:52.499ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
151522023-09-22T23:30:52.504Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.177789561Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1634}
151532023-09-22T23:30:52.508Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.178111909Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1634}
151542023-09-22T23:30:52.512Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.178134977Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1634}
151552023-09-22T23:30:52.517ZError: Difference in extent metadata found!
151562023-09-22T23:30:52.521Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.206392003Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1634}
151572023-09-22T23:30:52.525Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.206426407Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1634}
151582023-09-22T23:30:52.529Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.206445479Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1634}
151592023-09-22T23:30:52.533Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.226068031Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1634}
151602023-09-22T23:30:52.537Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.226099551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1634}
151612023-09-22T23:30:52.540Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:52.226118515Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1634}
151622023-09-22T23:30:52.545ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
151632023-09-22T23:30:52.549Z 0 000-019 106 106 106 162 162 162 F F F
151642023-09-22T23:30:52.553Z 1 020-039 111 111 111 168 168 168 F F F
151652023-09-22T23:30:52.556Z 2 040-059 108 108 108 165 165 165 F F F
151662023-09-22T23:30:52.560Z 3 060-079 113 113 113 170 170 170 T T F <---
151672023-09-22T23:30:52.564Z 4 080-099 113 113 113 170 170 170 F F F
151682023-09-22T23:30:52.568Z 5 100-119 113 113 113 170 170 170 F F F
151692023-09-22T23:30:52.571Z 6 120-139 110 110 110 167 167 167 T T F <---
151702023-09-22T23:30:52.577Z 7 140-159 108 108 108 165 165 165 F F F
151712023-09-22T23:30:52.581Z 8 160-179 114 114 112 171 171 169 F F F <---
151722023-09-22T23:30:52.585Z 9 180-199 114 114 111 171 171 168 F F F <---
151732023-09-22T23:30:52.588Z 10 200-219 114 114 105 171 171 160 F F F <---
151742023-09-22T23:30:52.592Z 11 220-239 114 114 107 171 171 164 F F F <---
151752023-09-22T23:30:52.596Z 12 240-259 114 114 110 171 171 167 F F T <---
151762023-09-22T23:30:52.600Z 13 260-279 113 113 113 170 170 170 F F F
151772023-09-22T23:30:52.604Z 14 280-299 110 110 110 166 166 166 F F F
151782023-09-22T23:30:52.607Z 15 300-319 110 110 110 166 166 166 F F F
151792023-09-22T23:30:52.611Z 16 320-339 114 114 105 171 171 160 F F F <---
151802023-09-22T23:30:52.614Z 17 340-359 113 113 113 170 170 170 F F F
151812023-09-22T23:30:52.617Z 18 360-379 110 110 110 166 166 166 F F F
151822023-09-22T23:30:52.621Z 19 380-399 108 108 108 165 165 165 T T F <---
151832023-09-22T23:30:52.625Z 20 400-419 110 110 110 167 167 167 F F F
151842023-09-22T23:30:52.629Z 21 420-439 114 114 110 171 171 166 F F F <---
151852023-09-22T23:30:52.633Z 22 440-459 114 114 108 171 171 165 F F F <---
151862023-09-22T23:30:52.636Z 23 460-479 114 114 106 171 171 163 F F F <---
151872023-09-22T23:30:52.640Z 24 480-499 114 114 108 171 171 165 F F F <---
151882023-09-22T23:30:52.644Z 25 500-519 114 114 113 171 171 170 F F F <---
151892023-09-22T23:30:52.648Z 26 520-539 108 108 108 165 165 165 T T F <---
151902023-09-22T23:30:52.652Z 27 540-559 113 113 113 170 170 170 T T F <---
151912023-09-22T23:30:52.655Z 28 560-579 113 113 113 170 170 170 T T F <---
151922023-09-22T23:30:52.660Z 29 580-599 108 108 108 165 165 165 F F F
151932023-09-22T23:30:52.723ZMax gen: 114, Max flush: 171
151942023-09-22T23:30:52.728ZA Difference in extent metadata is expected here
151952023-09-22T23:30:52.732ZOn loop 57
151962023-09-22T23:30:52.736Z
151972023-09-22T23:30:52.741ZVerifying data now
151982023-09-22T23:30:59.003ZNo change after verify
151992023-09-22T23:30:59.006ZLoop: 57 Downstairs dump after verify (and repair):
152002023-09-22T23:30:59.022Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.02188513Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1639}
152012023-09-22T23:30:59.025Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.022188924Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1639}
152022023-09-22T23:30:59.027Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.022213566Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1639}
152032023-09-22T23:30:59.041Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.04135Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1639}
152042023-09-22T23:30:59.043Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.041403304Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1639}
152052023-09-22T23:30:59.047Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.041424343Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1639}
152062023-09-22T23:30:59.052Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.052481048Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1639}
152072023-09-22T23:30:59.056Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.052533432Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1639}
152082023-09-22T23:30:59.060Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.052571875Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1639}
152092023-09-22T23:30:59.063ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
152102023-09-22T23:30:59.066Z 0 000-019 106 106 106 162 162 162 F F F
152112023-09-22T23:30:59.069Z 1 020-039 111 111 111 168 168 168 F F F
152122023-09-22T23:30:59.072Z 2 040-059 108 108 108 165 165 165 F F F
152132023-09-22T23:30:59.075Z 3 060-079 115 115 115 172 172 172 F F F
152142023-09-22T23:30:59.078Z 4 080-099 113 113 113 170 170 170 F F F
152152023-09-22T23:30:59.082Z 5 100-119 113 113 113 170 170 170 F F F
152162023-09-22T23:30:59.085Z 6 120-139 115 115 115 172 172 172 F F F
152172023-09-22T23:30:59.087Z 7 140-159 108 108 108 165 165 165 F F F
152182023-09-22T23:30:59.091Z 8 160-179 114 114 114 171 171 171 F F F
152192023-09-22T23:30:59.094Z 9 180-199 114 114 114 171 171 171 F F F
152202023-09-22T23:30:59.097Z 10 200-219 114 114 114 171 171 171 F F F
152212023-09-22T23:30:59.100Z 11 220-239 114 114 114 171 171 171 F F F
152222023-09-22T23:30:59.103Z 12 240-259 114 114 114 171 171 171 F F F
152232023-09-22T23:30:59.106Z 13 260-279 113 113 113 170 170 170 F F F
152242023-09-22T23:30:59.109Z 14 280-299 110 110 110 166 166 166 F F F
152252023-09-22T23:30:59.112Z 15 300-319 110 110 110 166 166 166 F F F
152262023-09-22T23:30:59.115Z 16 320-339 114 114 114 171 171 171 F F F
152272023-09-22T23:30:59.118Z 17 340-359 113 113 113 170 170 170 F F F
152282023-09-22T23:30:59.121Z 18 360-379 110 110 110 166 166 166 F F F
152292023-09-22T23:30:59.124Z 19 380-399 115 115 115 172 172 172 F F F
152302023-09-22T23:30:59.127Z 20 400-419 110 110 110 167 167 167 F F F
152312023-09-22T23:30:59.130Z 21 420-439 114 114 114 171 171 171 F F F
152322023-09-22T23:30:59.133Z 22 440-459 114 114 114 171 171 171 F F F
152332023-09-22T23:30:59.136Z 23 460-479 114 114 114 171 171 171 F F F
152342023-09-22T23:30:59.139Z 24 480-499 114 114 114 171 171 171 F F F
152352023-09-22T23:30:59.142Z 25 500-519 114 114 114 171 171 171 F F F
152362023-09-22T23:30:59.145Z 26 520-539 115 115 115 172 172 172 F F F
152372023-09-22T23:30:59.148Z 27 540-559 115 115 115 172 172 172 F F F
152382023-09-22T23:30:59.151Z 28 560-579 115 115 115 172 172 172 F F F
152392023-09-22T23:30:59.154Z 29 580-599 108 108 108 165 165 165 F F F
152402023-09-22T23:30:59.156ZMax gen: 115, Max flush: 172
152412023-09-22T23:30:59.159Z
152422023-09-22T23:30:59.162ZBegin loop 58
152432023-09-22T23:30:59.165ZDownstairs to restart: 2
152442023-09-22T23:30:59.168Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.100854544Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152452023-09-22T23:30:59.171Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.101233559Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152462023-09-22T23:30:59.174Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.101285969Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152472023-09-22T23:30:59.177Z{"msg":"Crucible stats registered with UUID: 8248d4ab-fdae-42e6-9ba3-98eb3bdfd4c8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.101334249Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152482023-09-22T23:30:59.179Z{"msg":"Crucible 8248d4ab-fdae-42e6-9ba3-98eb3bdfd4c8 has session id: 1b79d8c7-a2de-422d-9614-d8353b8e4b0b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.101402121Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152492023-09-22T23:30:59.182ZCrucible runtime is spawned
152502023-09-22T23:30:59.185ZThe guest has requested activation with gen:116
152512023-09-22T23:30:59.188Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.101751397Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641,"task":"up_listen"}
152522023-09-22T23:30:59.191Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.101823116Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152532023-09-22T23:30:59.194Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.101867982Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152542023-09-22T23:30:59.199Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.101907532Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641,"looper":"0"}
152552023-09-22T23:30:59.203Z{"msg":"[0] 8248d4ab-fdae-42e6-9ba3-98eb3bdfd4c8 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.102137438Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641,"looper":"0"}
152562023-09-22T23:30:59.206Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.102263505Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641,"looper":"1"}
152572023-09-22T23:30:59.209Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.102420109Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641,"looper":"2"}
152582023-09-22T23:30:59.212Z{"msg":"[1] 8248d4ab-fdae-42e6-9ba3-98eb3bdfd4c8 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.102500938Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641,"looper":"1"}
152592023-09-22T23:30:59.215Z{"msg":"8248d4ab-fdae-42e6-9ba3-98eb3bdfd4c8 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.102829665Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152602023-09-22T23:30:59.218Z{"msg":"Set desired generation to :116","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.102880839Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152612023-09-22T23:30:59.221Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.103008939Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152622023-09-22T23:30:59.224Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.103330399Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152632023-09-22T23:30:59.227Z{"msg":"[0] 8248d4ab-fdae-42e6-9ba3-98eb3bdfd4c8 (72cc369c-b513-47b7-9c68-7c355ee76de0) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.10373921Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152642023-09-22T23:30:59.230Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.103854489Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152652023-09-22T23:30:59.233Z{"msg":"[0] client is_active_req TRUE, promote! session 72cc369c-b513-47b7-9c68-7c355ee76de0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.103937457Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152662023-09-22T23:30:59.236Z{"msg":"[1] 8248d4ab-fdae-42e6-9ba3-98eb3bdfd4c8 (72cc369c-b513-47b7-9c68-7c355ee76de0) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.104009818Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152672023-09-22T23:30:59.239Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.104119994Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152682023-09-22T23:30:59.242Z{"msg":"[1] client is_active_req TRUE, promote! session 72cc369c-b513-47b7-9c68-7c355ee76de0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.104162206Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152692023-09-22T23:30:59.245Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.104424636Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152702023-09-22T23:30:59.248Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.104510682Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152712023-09-22T23:30:59.251Z{"msg":"8248d4ab-fdae-42e6-9ba3-98eb3bdfd4c8 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.104553581Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152722023-09-22T23:30:59.254Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.104668063Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152732023-09-22T23:30:59.258Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.104723029Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152742023-09-22T23:30:59.261Z{"msg":"8248d4ab-fdae-42e6-9ba3-98eb3bdfd4c8 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.104747172Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152752023-09-22T23:30:59.265Z{"msg":"[0] 8248d4ab-fdae-42e6-9ba3-98eb3bdfd4c8 (72cc369c-b513-47b7-9c68-7c355ee76de0) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.107509664Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152762023-09-22T23:30:59.268Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.107683047Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152772023-09-22T23:30:59.271Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:30:59.107729212Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152782023-09-22T23:30:59.274Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.107760549Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152792023-09-22T23:30:59.278Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.107803131Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152802023-09-22T23:30:59.301Z{"msg":"8248d4ab-fdae-42e6-9ba3-98eb3bdfd4c8 WaitQuorum WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.10782429Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152812023-09-22T23:30:59.304Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.107861332Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152822023-09-22T23:30:59.307Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.107900021Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152832023-09-22T23:30:59.310Z{"msg":"[1] 8248d4ab-fdae-42e6-9ba3-98eb3bdfd4c8 (72cc369c-b513-47b7-9c68-7c355ee76de0) WaitQuorum WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.107942143Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152842023-09-22T23:30:59.313Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.10796427Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152852023-09-22T23:30:59.318Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:30:59.108000814Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152862023-09-22T23:30:59.323Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.108020597Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152872023-09-22T23:30:59.326Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.10805733Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152882023-09-22T23:30:59.330Z{"msg":"8248d4ab-fdae-42e6-9ba3-98eb3bdfd4c8 WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.108095637Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152892023-09-22T23:30:59.334Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.108134987Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152902023-09-22T23:30:59.337Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:30:59.108154245Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152912023-09-22T23:31:00.103Z{"msg":"[2] 8248d4ab-fdae-42e6-9ba3-98eb3bdfd4c8 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.103194152Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641,"looper":"2"}
152922023-09-22T23:31:00.106Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.103283528Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152932023-09-22T23:31:00.109Z{"msg":"[2] 8248d4ab-fdae-42e6-9ba3-98eb3bdfd4c8 (72cc369c-b513-47b7-9c68-7c355ee76de0) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.104331851Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152942023-09-22T23:31:00.112Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.104393008Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152952023-09-22T23:31:00.116Z{"msg":"[2] client is_active_req TRUE, promote! session 72cc369c-b513-47b7-9c68-7c355ee76de0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.104437192Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152962023-09-22T23:31:00.119Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.105062684Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152972023-09-22T23:31:00.122Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.105122487Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152982023-09-22T23:31:00.125Z{"msg":"8248d4ab-fdae-42e6-9ba3-98eb3bdfd4c8 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.105166157Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
152992023-09-22T23:31:00.130Z{"msg":"[2] 8248d4ab-fdae-42e6-9ba3-98eb3bdfd4c8 (72cc369c-b513-47b7-9c68-7c355ee76de0) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.107565751Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153002023-09-22T23:31:00.133Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.107627362Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153012023-09-22T23:31:00.136Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:31:00.107684787Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153022023-09-22T23:31:00.139Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.107739868Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153032023-09-22T23:31:00.142Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.107795022Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153042023-09-22T23:31:00.146Z{"msg":"8248d4ab-fdae-42e6-9ba3-98eb3bdfd4c8 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.107850791Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153052023-09-22T23:31:00.149Z{"msg":"[0]R flush_numbers[0..12]: [162, 168, 165, 172, 170, 170, 172, 165, 171, 171, 171, 171]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.107907252Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153062023-09-22T23:31:00.152Z{"msg":"[0]R generation[0..12]: [106, 111, 108, 115, 113, 113, 115, 108, 114, 114, 114, 114]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:116
153072023-09-22T23:31:00.155ZWait for a query_work_queue command to finish before sending IO
153082023-09-22T23:31:00.159Z,"time":"2023-09-22T23:31:00.107963385Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153092023-09-22T23:31:00.162Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.108177619Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153102023-09-22T23:31:00.165Z{"msg":"[1]R flush_numbers[0..12]: [162, 168, 165, 172, 170, 170, 172, 165, 171, 171, 171, 171]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.108251992Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153112023-09-22T23:31:00.168Z{"msg":"[1]R generation[0..12]: [106, 111, 108, 115, 113, 113, 115, 108, 114, 114, 114, 114]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.108300055Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153122023-09-22T23:31:00.171Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.108343337Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153132023-09-22T23:31:00.175Z{"msg":"[2]R flush_numbers[0..12]: [162, 168, 165, 172, 170, 170, 172, 165, 171, 171, 171, 171]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.108384221Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153142023-09-22T23:31:00.178Z{"msg":"[2]R generation[0..12]: [106, 111, 108, 115, 113, 113, 115, 108, 114, 114, 114, 114]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.108424202Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153152023-09-22T23:31:00.181Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.108477996Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153162023-09-22T23:31:00.185Z{"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.108572854Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153172023-09-22T23:31:00.188Z{"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.108621235Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153182023-09-22T23:31:00.191Z{"msg":"Next flush: 173","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.108663696Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153192023-09-22T23:31:00.194Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.108702428Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153202023-09-22T23:31:00.197Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.108743237Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153212023-09-22T23:31:00.202Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.108782753Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153222023-09-22T23:31:00.205Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.108829393Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153232023-09-22T23:31:00.209Z{"msg":"8248d4ab-fdae-42e6-9ba3-98eb3bdfd4c8 is now active with session: 72cc369c-b513-47b7-9c68-7c355ee76de0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.108870077Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153242023-09-22T23:31:00.212Z{"msg":"8248d4ab-fdae-42e6-9ba3-98eb3bdfd4c8 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.108911753Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153252023-09-22T23:31:00.215Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
153262023-09-22T23:31:00.219Z,"time":"2023-09-22T23:31:00.108956799Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153272023-09-22T23:31:00.222Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.109058427Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153282023-09-22T23:31:00.225Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.109104942Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153292023-09-22T23:31:00.228Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.109144389Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153302023-09-22T23:31:00.231Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.109182896Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153312023-09-22T23:31:00.235Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.109221952Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153322023-09-22T23:31:00.238Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.109260594Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153332023-09-22T23:31:00.241Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.109298838Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153342023-09-22T23:31:00.244Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.109336751Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153352023-09-22T23:31:00.247Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.109374533Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153362023-09-22T23:31:00.250Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.109413413Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1641}
153372023-09-22T23:31:00.253ZLoading write count information from file "/tmp/test_repair_verify.data"
153382023-09-22T23:31:00.256ZRun Repair workload
153392023-09-22T23:31:00.259Z01/30 Write block 148 len 40960 data: 9 10 10 10 9 6 7 8 7 9
153402023-09-22T23:31:00.262Z02/30 Read block 379 len 28672
153412023-09-22T23:31:00.266Z03/30 Write block 338 len 4096 data: 7
153422023-09-22T23:31:00.269Z04/30 Write block 45 len 40960 data: 9 8 9 8 8 7 8 6 4 3
153432023-09-22T23:31:00.272Z05/30 Read block 514 len 36864
153442023-09-22T23:31:00.275Z06/30 Write block 273 len 16384 data: 14 14 15 18
153452023-09-22T23:31:00.279Z07/30 Read block 588 len 40960
153462023-09-22T23:31:00.282Z08/30 Read block 16 len 24576
153472023-09-22T23:31:00.285Z09/30 Write block 29 len 32768 data: 5 5 6 5 4 8 10 13
153482023-09-22T23:31:00.289Z10/30 Read block 463 len 16384
153492023-09-22T23:31:00.292Z11/30 Write block 225 len 20480 data: 10 10 9 8 7
153502023-09-22T23:31:00.295Z12/30 Write block 542 len 32768 data: 8 9 8 7 10 7 11 14
153512023-09-22T23:31:00.299Z13/30 Write block 545 len 12288 data: 8 11 8
153522023-09-22T23:31:00.323Z14/30 Read block 318 len 12288
153532023-09-22T23:31:00.327Z15/30 Write block 187 len 4096 data: 9
153542023-09-22T23:31:00.330Z16/30 Flush
153552023-09-22T23:31:00.333Z17/30 Flush
153562023-09-22T23:31:00.336Z18/30 Write block 203 len 12288 data: 12 11 16
153572023-09-22T23:31:00.339Z19/30 Read block 445 len 20480
153582023-09-22T23:31:00.342Z20/30 Write block 292 len 4096 data: 15
153592023-09-22T23:31:00.345Z21/30 Read block 479 len 8192
153602023-09-22T23:31:00.348Z22/30 Write block 470 len 12288 data: 8 6 7
153612023-09-22T23:31:00.351Z23/30 Read block 377 len 4096
153622023-09-22T23:31:00.354Z24/30 Read block 418 len 32768
153632023-09-22T23:31:00.358Z25/30 Write block 381 len 16384 data: 9 8 8 8
153642023-09-22T23:31:00.361Z26/30 Read block 63 len 20480
153652023-09-22T23:31:00.364Z27/30 Read block 174 len 24576
153662023-09-22T23:31:00.368Z28/30 Read block 297 len 28672
153672023-09-22T23:31:00.371Z29/30 Read block 199 len 32768
153682023-09-22T23:31:00.374Z30/30 Write block 11 len 28672 data: 8 9 10 8 9 8 8
153692023-09-22T23:31:00.377Z----------------------------------------------------------------
153702023-09-22T23:31:00.380Z Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30
153712023-09-22T23:31:00.384ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
153722023-09-22T23:31:00.388Z 1 Acked 1000 Write 10 Done Done Done false
153732023-09-22T23:31:00.391Z 2 Acked 1001 Read 7 Done Done Done false
153742023-09-22T23:31:00.395Z 3 Acked 1002 Write 1 Done Done Done false
153752023-09-22T23:31:00.398Z 4 Acked 1003 Write 10 Done Done Done false
153762023-09-22T23:31:00.401Z 5 Acked 1004 Read 9 Done Done Done false
153772023-09-22T23:31:00.404Z 6 Acked 1005 Write 4 Done Done Done false
153782023-09-22T23:31:00.407Z 7 Acked 1006 Read 10 Done Done Sent false
153792023-09-22T23:31:00.411Z 8 Acked 1007 Read 6 Done Done Sent false
153802023-09-22T23:31:00.415Z 9 Acked 1008 Write 8 Done Done Sent false
153812023-09-22T23:31:00.418Z 10 Acked 1009 Read 4 Done Done Sent false
153822023-09-22T23:31:00.421Z 11 Acked 1010 Write 5 Done Done Sent false
153832023-09-22T23:31:00.424Z 12 Acked 1011 Write 8 Done Done Sent false
153842023-09-22T23:31:00.427Z 13 Acked 1012 Write 3 Done Done Sent false
153852023-09-22T23:31:00.430Z 14 Acked 1013 Read 3 Done Done Sent false
153862023-09-22T23:31:00.433Z 15 Acked 1014 Write 1 Done Done Sent false
153872023-09-22T23:31:00.436Z 16 Acked 1015 Flush 0 Done Done Sent false
153882023-09-22T23:31:00.439Z 17 Acked 1016 Flush 0 Done Done Sent false
153892023-09-22T23:31:00.442Z 18 Acked 1017 Write 3 Done Done Sent false
153902023-09-22T23:31:00.445Z 19 Acked 1018 Read 5 Done Done Sent false
153912023-09-22T23:31:00.448Z 20 Acked 1019 Write 1 Done Done Sent false
153922023-09-22T23:31:00.451Z 21 Acked 1020 Read 2 Done Done Sent false
153932023-09-22T23:31:00.454Z 22 Acked 1021 Write 3 Done Done Sent false
153942023-09-22T23:31:00.457Z 23 Acked 1022 Read 1 Done Done Sent false
153952023-09-22T23:31:00.465Z 24 Acked 1023 Read 8 Done Done Sent false
153962023-09-22T23:31:00.468Z 25 Acked 1024 Write 4 Done Done Sent false
153972023-09-22T23:31:00.474Z 26 Acked 1025 Read 5 Done Done Sent false
153982023-09-22T23:31:00.477Z 27 Acked 1026 Read 6 Done Done Sent false
153992023-09-22T23:31:00.481Z 28 Acked 1027 Read 7 Done Done Sent false
154002023-09-22T23:31:00.485Z 29 Acked 1028 Read 8 Done Done Sent false
154012023-09-22T23:31:00.488Z 30 Acked 1029 Write 7 Sent Sent Sent false
154022023-09-22T23:31:00.491Z STATES DS:0 DS:1 DS:2 TOTAL
154032023-09-22T23:31:00.494Z New 0 0 0 0
154042023-09-22T23:31:00.498Z Sent 1 1 24 26
154052023-09-22T23:31:00.500Z Done 29 29 6 64
154062023-09-22T23:31:00.503Z Skipped 0 0 0 0
154072023-09-22T23:31:00.506Z Error 0 0 0 0
154082023-09-22T23:31:00.509ZLast Flush: 1016 1016 0
154092023-09-22T23:31:00.513ZDownstairs last five completed:
154102023-09-22T23:31:00.516ZUpstairs last five completed: 30 29 28 27 26 25
154112023-09-22T23:31:00.519ZWrote out file "/tmp/test_repair_verify.data"
154122023-09-22T23:31:00.523Z
154132023-09-22T23:31:00.527ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
154142023-09-22T23:31:00.531Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.240300664Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1642}
154152023-09-22T23:31:00.534Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.240635823Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1642}
154162023-09-22T23:31:00.537Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.240659248Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1642}
154172023-09-22T23:31:00.540Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.268013834Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1642}
154182023-09-22T23:31:00.543ZError: Difference in extent metadata found!
154192023-09-22T23:31:00.546Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.268048274Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1642}
154202023-09-22T23:31:00.550Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.26806748Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1642}
154212023-09-22T23:31:00.553Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.286112479Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1642}
154222023-09-22T23:31:00.556Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.286143355Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1642}
154232023-09-22T23:31:00.559Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:00.286162411Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1642}
154242023-09-22T23:31:00.562ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
154252023-09-22T23:31:00.565Z 0 000-019 106 106 106 162 162 162 T T F <---
154262023-09-22T23:31:00.569Z 1 020-039 116 116 111 173 173 168 F F F <---
154272023-09-22T23:31:00.572Z 2 040-059 116 116 108 173 173 165 F F T <---
154282023-09-22T23:31:00.576Z 3 060-079 115 115 115 172 172 172 F F F
154292023-09-22T23:31:00.579Z 4 080-099 113 113 113 170 170 170 F F F
154302023-09-22T23:31:00.582Z 5 100-119 113 113 113 170 170 170 F F F
154312023-09-22T23:31:00.585Z 6 120-139 115 115 115 172 172 172 F F F
154322023-09-22T23:31:00.588Z 7 140-159 116 116 108 173 173 165 F F T <---
154332023-09-22T23:31:00.592Z 8 160-179 114 114 114 171 171 171 F F F
154342023-09-22T23:31:00.595Z 9 180-199 116 116 114 173 173 171 F F F <---
154352023-09-22T23:31:00.598Z 10 200-219 114 114 114 171 171 171 T T F <---
154362023-09-22T23:31:00.602Z 11 220-239 116 116 114 173 173 171 F F F <---
154372023-09-22T23:31:00.605Z 12 240-259 114 114 114 171 171 171 F F F
154382023-09-22T23:31:00.608Z 13 260-279 116 116 113 173 173 170 F F T <---
154392023-09-22T23:31:00.611Z 14 280-299 110 110 110 166 166 166 T T F <---
154402023-09-22T23:31:00.614Z 15 300-319 110 110 110 166 166 166 F F F
154412023-09-22T23:31:00.618Z 16 320-339 116 116 114 173 173 171 F F T <---
154422023-09-22T23:31:00.621Z 17 340-359 113 113 113 170 170 170 F F F
154432023-09-22T23:31:00.624Z 18 360-379 110 110 110 166 166 166 F F F
154442023-09-22T23:31:00.627Z 19 380-399 115 115 115 172 172 172 T T F <---
154452023-09-22T23:31:00.631Z 20 400-419 110 110 110 167 167 167 F F F
154462023-09-22T23:31:00.635Z 21 420-439 114 114 114 171 171 171 F F F
154472023-09-22T23:31:00.638Z 22 440-459 114 114 114 171 171 171 F F F
154482023-09-22T23:31:00.641Z 23 460-479 114 114 114 171 171 171 T T F <---
154492023-09-22T23:31:00.644Z 24 480-499 114 114 114 171 171 171 F F F
154502023-09-22T23:31:00.648Z 25 500-519 114 114 114 171 171 171 F F F
154512023-09-22T23:31:00.651Z 26 520-539 115 115 115 172 172 172 F F F
154522023-09-22T23:31:00.654Z 27 540-559 116 116 115 173 173 172 F F F <---
154532023-09-22T23:31:00.657Z 28 560-579 115 115 115 172 172 172 F F F
154542023-09-22T23:31:00.661Z 29 580-599 108 108 108 165 165 165 F F F
154552023-09-22T23:31:00.665ZMax gen: 116, Max flush: 173
154562023-09-22T23:31:00.668ZA Difference in extent metadata is expected here
154572023-09-22T23:31:00.671ZOn loop 58
154582023-09-22T23:31:00.674Z
154592023-09-22T23:31:00.679ZVerifying data now
154602023-09-22T23:31:06.728ZNo change after verify
154612023-09-22T23:31:06.732ZLoop: 58 Downstairs dump after verify (and repair):
154622023-09-22T23:31:06.746Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.746672275Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1647}
154632023-09-22T23:31:06.749Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.747070554Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1647}
154642023-09-22T23:31:06.753Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.747095365Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1647}
154652023-09-22T23:31:06.765Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.765997508Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1647}
154662023-09-22T23:31:06.768Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.766058839Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1647}
154672023-09-22T23:31:06.771Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.766079637Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1647}
154682023-09-22T23:31:06.776Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.777135877Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1647}
154692023-09-22T23:31:06.779Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.777197584Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1647}
154702023-09-22T23:31:06.782Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.777218214Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1647}
154712023-09-22T23:31:06.787ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
154722023-09-22T23:31:06.790Z 0 000-019 117 117 117 174 174 174 F F F
154732023-09-22T23:31:06.793Z 1 020-039 116 116 116 173 173 173 F F F
154742023-09-22T23:31:06.796Z 2 040-059 116 116 116 173 173 173 F F F
154752023-09-22T23:31:06.799Z 3 060-079 115 115 115 172 172 172 F F F
154762023-09-22T23:31:06.802Z 4 080-099 113 113 113 170 170 170 F F F
154772023-09-22T23:31:06.805Z 5 100-119 113 113 113 170 170 170 F F F
154782023-09-22T23:31:06.807Z 6 120-139 115 115 115 172 172 172 F F F
154792023-09-22T23:31:06.811Z 7 140-159 116 116 116 173 173 173 F F F
154802023-09-22T23:31:06.817Z 8 160-179 114 114 114 171 171 171 F F F
154812023-09-22T23:31:06.821Z 9 180-199 116 116 116 173 173 173 F F F
154822023-09-22T23:31:06.825Z 10 200-219 117 117 117 174 174 174 F F F
154832023-09-22T23:31:06.828Z 11 220-239 116 116 116 173 173 173 F F F
154842023-09-22T23:31:06.831Z 12 240-259 114 114 114 171 171 171 F F F
154852023-09-22T23:31:06.834Z 13 260-279 116 116 116 173 173 173 F F F
154862023-09-22T23:31:06.837Z 14 280-299 117 117 117 174 174 174 F F F
154872023-09-22T23:31:06.840Z 15 300-319 110 110 110 166 166 166 F F F
154882023-09-22T23:31:06.843Z 16 320-339 116 116 116 173 173 173 F F F
154892023-09-22T23:31:06.846Z 17 340-359 113 113 113 170 170 170 F F F
154902023-09-22T23:31:06.849Z 18 360-379 110 110 110 166 166 166 F F F
154912023-09-22T23:31:06.852Z 19 380-399 117 117 117 174 174 174 F F F
154922023-09-22T23:31:06.855Z 20 400-419 110 110 110 167 167 167 F F F
154932023-09-22T23:31:06.858Z 21 420-439 114 114 114 171 171 171 F F F
154942023-09-22T23:31:06.861Z 22 440-459 114 114 114 171 171 171 F F F
154952023-09-22T23:31:06.864Z 23 460-479 117 117 117 174 174 174 F F F
154962023-09-22T23:31:06.868Z 24 480-499 114 114 114 171 171 171 F F F
154972023-09-22T23:31:06.871Z 25 500-519 114 114 114 171 171 171 F F F
154982023-09-22T23:31:06.874Z 26 520-539 115 115 115 172 172 172 F F F
154992023-09-22T23:31:06.880Z 27 540-559 116 116 116 173 173 173 F F F
155002023-09-22T23:31:06.883Z 28 560-579 115 115 115 172 172 172 F F F
155012023-09-22T23:31:06.885Z 29 580-599 108 108 108 165 165 165 F F F
155022023-09-22T23:31:06.888ZMax gen: 117, Max flush: 174
155032023-09-22T23:31:06.892Z
155042023-09-22T23:31:06.896ZBegin loop 59
155052023-09-22T23:31:06.899ZDownstairs to restart: 1
155062023-09-22T23:31:06.902Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.83105482Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155072023-09-22T23:31:06.905Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.831446797Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155082023-09-22T23:31:06.908Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.831500464Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155092023-09-22T23:31:06.911Z{"msg":"Crucible stats registered with UUID: 13a67981-724d-4053-bbd1-60a785dc5b47","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.831566348Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155102023-09-22T23:31:06.914Z{"msg":"Crucible 13a67981-724d-4053-bbd1-60a785dc5b47 has session id: 40368ea3-598e-4ce3-a1d4-a7bf6dc47db9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.831634942Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155112023-09-22T23:31:06.917ZCrucible runtime is spawned
155122023-09-22T23:31:06.920ZThe guest has requested activation with gen:118
155132023-09-22T23:31:06.923Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.831887525Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649,"task":"up_listen"}
155142023-09-22T23:31:06.926Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.8319579Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155152023-09-22T23:31:06.929Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.832017949Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155162023-09-22T23:31:06.932Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.832096217Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649,"looper":"0"}
155172023-09-22T23:31:06.935Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.832340835Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649,"looper":"2"}
155182023-09-22T23:31:06.938Z{"msg":"[0] 13a67981-724d-4053-bbd1-60a785dc5b47 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.832455116Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649,"looper":"0"}
155192023-09-22T23:31:06.941Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.83249924Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649,"looper":"1"}
155202023-09-22T23:31:06.944Z{"msg":"[2] 13a67981-724d-4053-bbd1-60a785dc5b47 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.8325951Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649,"looper":"2"}
155212023-09-22T23:31:06.947Z{"msg":"13a67981-724d-4053-bbd1-60a785dc5b47 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.832958593Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155222023-09-22T23:31:06.951Z{"msg":"Set desired generation to :118","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.833009401Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155232023-09-22T23:31:06.954Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.833105137Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155242023-09-22T23:31:06.957Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.83343222Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155252023-09-22T23:31:06.960Z{"msg":"[0] 13a67981-724d-4053-bbd1-60a785dc5b47 (a3e33ebd-c5ba-4d5d-aae1-1356afe1e0be) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.833821106Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155262023-09-22T23:31:06.963Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.83387473Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155272023-09-22T23:31:06.966Z{"msg":"[0] client is_active_req TRUE, promote! session a3e33ebd-c5ba-4d5d-aae1-1356afe1e0be","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.833917748Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155282023-09-22T23:31:06.969Z{"msg":"[2] 13a67981-724d-4053-bbd1-60a785dc5b47 (a3e33ebd-c5ba-4d5d-aae1-1356afe1e0be) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.834083585Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155292023-09-22T23:31:06.972Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.834158275Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155302023-09-22T23:31:06.975Z{"msg":"[2] client is_active_req TRUE, promote! session a3e33ebd-c5ba-4d5d-aae1-1356afe1e0be","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.834193388Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155312023-09-22T23:31:06.978Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.834523491Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155322023-09-22T23:31:06.981Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.834609167Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155332023-09-22T23:31:06.984Z{"msg":"13a67981-724d-4053-bbd1-60a785dc5b47 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.834681449Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155342023-09-22T23:31:06.987Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.834753589Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155352023-09-22T23:31:06.990Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.834829825Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155362023-09-22T23:31:06.993Z{"msg":"13a67981-724d-4053-bbd1-60a785dc5b47 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.834974782Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155372023-09-22T23:31:06.996Z{"msg":"[0] 13a67981-724d-4053-bbd1-60a785dc5b47 (a3e33ebd-c5ba-4d5d-aae1-1356afe1e0be) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.83678487Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155382023-09-22T23:31:06.999Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.836851859Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155392023-09-22T23:31:07.003Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:31:06.836902223Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155402023-09-22T23:31:07.006Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.836940551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155412023-09-22T23:31:07.009Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.837008147Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155422023-09-22T23:31:07.012Z{"msg":"13a67981-724d-4053-bbd1-60a785dc5b47 WaitQuorum New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.837056006Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155432023-09-22T23:31:07.015Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.837096141Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155442023-09-22T23:31:07.018Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.837135162Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155452023-09-22T23:31:07.021Z{"msg":"[2] 13a67981-724d-4053-bbd1-60a785dc5b47 (a3e33ebd-c5ba-4d5d-aae1-1356afe1e0be) WaitQuorum New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.837176452Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155462023-09-22T23:31:07.025Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.837197528Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155472023-09-22T23:31:07.028Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:31:06.83723416Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155482023-09-22T23:31:07.031Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.837273816Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155492023-09-22T23:31:07.034Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.83731231Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155502023-09-22T23:31:07.037Z{"msg":"13a67981-724d-4053-bbd1-60a785dc5b47 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.83735136Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155512023-09-22T23:31:07.040Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.837389911Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155522023-09-22T23:31:07.050Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:06.837409882Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155532023-09-22T23:31:07.835Z{"msg":"[1] 13a67981-724d-4053-bbd1-60a785dc5b47 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.834322083Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649,"looper":"1"}
155542023-09-22T23:31:07.838Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.834409331Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155552023-09-22T23:31:07.842Z{"msg":"[1] 13a67981-724d-4053-bbd1-60a785dc5b47 (a3e33ebd-c5ba-4d5d-aae1-1356afe1e0be) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.835484326Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155562023-09-22T23:31:07.845Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.835539956Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155572023-09-22T23:31:07.848Z{"msg":"[1] client is_active_req TRUE, promote! session a3e33ebd-c5ba-4d5d-aae1-1356afe1e0be","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.835583729Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155582023-09-22T23:31:07.851Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.836232745Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155592023-09-22T23:31:07.854Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.836291373Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155602023-09-22T23:31:07.858Z{"msg":"13a67981-724d-4053-bbd1-60a785dc5b47 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.836334187Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155612023-09-22T23:31:07.861Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.838500804Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155622023-09-22T23:31:07.864Z{"msg":"[1] 13a67981-724d-4053-bbd1-60a785dc5b47 (a3e33ebd-c5ba-4d5d-aae1-1356afe1e0be) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.838710208Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155632023-09-22T23:31:07.867Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.838773922Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155642023-09-22T23:31:07.870Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:31:07.838815602Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155652023-09-22T23:31:07.874Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.838836418Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155662023-09-22T23:31:07.877Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.838854573Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155672023-09-22T23:31:07.880Z{"msg":"13a67981-724d-4053-bbd1-60a785dc5b47 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.838873081Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155682023-09-22T23:31:07.883Z{"msg":"[0]R flush_numbers[0..12]: [174, 173, 173, 172, 170, 170, 172, 173, 171, 173, 174, 173]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.838911883Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155692023-09-22T23:31:07.886Z{"msg":"[0]R generation[0..12]: [117, 116, 116, 115, 113, 113, 115, 116, 114, 116, 117, 116]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.838964079Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155702023-09-22T23:31:07.889Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.839032271Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155712023-09-22T23:31:07.892Z{"msg":"[1]R flush_numbers[0..12]: [174, 173, 173, 172, 170, 170, 172, 173, 171, 173, 174, 173]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.839111312Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155722023-09-22T23:31:07.896Z{The guest has finished waiting for activation with:118
155732023-09-22T23:31:07.899Z"msg":"Wait for a query_work_queue command to finish before sending IO
155742023-09-22T23:31:07.901Z[1]R generation[0..12]: [117, 116, 116, 115, 113, 113, 115, 116, 114, 116, 117, 116]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.839176547Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155752023-09-22T23:31:07.904Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.839203506Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155762023-09-22T23:31:07.908Z{"msg":"[2]R flush_numbers[0..12]: [174, 173, 173, 172, 170, 170, 172, 173, 171, 173, 174, 173]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.839252587Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155772023-09-22T23:31:07.911Z{"msg":"[2]R generation[0..12]: [117, 116, 116, 115, 113, 113, 115, 116, 114, 116, 117, 116]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.839317539Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155782023-09-22T23:31:07.914Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.839344851Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155792023-09-22T23:31:07.917Z{"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.8393684Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155802023-09-22T23:31:07.921Z{"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.839409673Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155812023-09-22T23:31:07.924Z{"msg":"Next flush: 175","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.839455665Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155822023-09-22T23:31:07.927Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.839523695Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155832023-09-22T23:31:07.930Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.839550695Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155842023-09-22T23:31:07.933Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.83961475Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155852023-09-22T23:31:07.936Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.839656231Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155862023-09-22T23:31:07.939Z{"msg":"13a67981-724d-4053-bbd1-60a785dc5b47 is now active with session: a3e33ebd-c5ba-4d5d-aae1-1356afe1e0be","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.83971877Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155872023-09-22T23:31:07.942Z{"msg":"13a67981-724d-4053-bbd1-60a785dc5b47 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.839764135Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155882023-09-22T23:31:07.945Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.839790498Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155892023-09-22T23:31:07.948Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.839853025Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155902023-09-22T23:31:07.952Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.839919266Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155912023-09-22T23:31:07.956Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.839965149Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155922023-09-22T23:31:07.959Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.840019487Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155932023-09-22T23:31:07.962Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.840059208Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155942023-09-22T23:31:07.965Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.840082278Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155952023-09-22T23:31:07.968ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
155962023-09-22T23:31:07.972Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.840144727Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155972023-09-22T23:31:07.975Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.840182521Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155982023-09-22T23:31:07.978Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.840213188Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
155992023-09-22T23:31:07.980Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.840275279Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1649}
156002023-09-22T23:31:07.983ZLoading write count information from file "/tmp/test_repair_verify.data"
156012023-09-22T23:31:07.987ZRun Repair workload
156022023-09-22T23:31:07.990Z01/30 Write block 71 len 40960 data: 10 10 7 8 9 9 11 11 14 12
156032023-09-22T23:31:07.993Z02/30 Flush
156042023-09-22T23:31:07.996Z03/30 Write block 317 len 28672 data: 8 7 7 9 10 7 9
156052023-09-22T23:31:07.999Z04/30 Read block 259 len 24576
156062023-09-22T23:31:08.002Z05/30 Read block 205 len 20480
156072023-09-22T23:31:08.005Z06/30 Write block 19 len 40960 data: 7 7 4 8 6 6 8 9 10 7
156082023-09-22T23:31:08.008Z07/30 Write block 392 len 36864 data: 9 7 9 11 10 10 11 9 12
156092023-09-22T23:31:08.011Z08/30 Write block 525 len 36864 data: 12 12 12 11 9 7 9 10 12
156102023-09-22T23:31:08.014Z09/30 Read block 454 len 20480
156112023-09-22T23:31:08.017Z10/30 Write block 302 len 8192 data: 10 11
156122023-09-22T23:31:08.019Z11/30 Flush
156132023-09-22T23:31:08.022Z12/30 Write block 537 len 20480 data: 13 14 14 10 7
156142023-09-22T23:31:08.025Z13/30 Read block 81 len 24576
156152023-09-22T23:31:08.028Z14/30 Read block 388 len 12288
156162023-09-22T23:31:08.031Z15/30 Read block 317 len 40960
156172023-09-22T23:31:08.034Z16/30 Read block 210 len 32768
156182023-09-22T23:31:08.037Z17/30 Write block 272 len 4096 data: 16
156192023-09-22T23:31:08.040Z18/30 Write block 3 len 32768 data: 7 6 7 7 8 8 8 8
156202023-09-22T23:31:08.043Z19/30 Write block 486 len 16384 data: 15 12 16 17
156212023-09-22T23:31:08.046Z20/30 Flush
156222023-09-22T23:31:08.049Z21/30 Write block 185 len 8192 data: 9 10
156232023-09-22T23:31:08.052Z22/30 Read block 528 len 40960
156242023-09-22T23:31:08.055Z23/30 Flush
156252023-09-22T23:31:08.058Z24/30 Write block 328 len 8192 data: 9 8
156262023-09-22T23:31:08.061Z25/30 Read block 186 len 28672
156272023-09-22T23:31:08.064Z26/30 Read block 486 len 32768
156282023-09-22T23:31:08.067Z27/30 Write block 218 len 16384 data: 9 7 9 8
156292023-09-22T23:31:08.071Z28/30 Write block 345 len 36864 data: 10 11 10 10 8 7 9 9 8
156302023-09-22T23:31:08.075Z29/30 Read block 180 len 12288
156312023-09-22T23:31:08.078Z30/30 Write block 194 len 12288 data: 9 11 11
156322023-09-22T23:31:08.081Z----------------------------------------------------------------
156332023-09-22T23:31:08.084Z Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:28
156342023-09-22T23:31:08.087ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
156352023-09-22T23:31:08.090Z 3 Acked 1002 Write 7 Done Done Done false
156362023-09-22T23:31:08.093Z 4 Acked 1003 Read 6 Done Done Done false
156372023-09-22T23:31:08.096Z 5 Acked 1004 Read 5 Done Sent Done false
156382023-09-22T23:31:08.098Z 6 Acked 1005 Write 10 Done Sent Done false
156392023-09-22T23:31:08.101Z 7 Acked 1006 Write 9 Done Sent Done false
156402023-09-22T23:31:08.104Z 8 Acked 1007 Write 9 Done Sent Done false
156412023-09-22T23:31:08.107Z 9 Acked 1008 Read 5 Done Sent Done false
156422023-09-22T23:31:08.110Z 10 Acked 1009 Write 2 Done Sent Done false
156432023-09-22T23:31:08.113Z 11 Acked 1010 Flush 0 Done Sent Done false
156442023-09-22T23:31:08.116Z 12 Acked 1011 Write 5 Done Sent Done false
156452023-09-22T23:31:08.119Z 13 Acked 1012 Read 6 Done Sent Done false
156462023-09-22T23:31:08.122Z 14 Acked 1013 Read 3 Done Sent Done false
156472023-09-22T23:31:08.125Z 15 Acked 1014 Read 10 Done Sent Done false
156482023-09-22T23:31:08.127Z 16 Acked 1015 Read 8 Done Sent Done false
156492023-09-22T23:31:08.130Z 17 Acked 1016 Write 1 Done Sent Done false
156502023-09-22T23:31:08.133Z 18 Acked 1017 Write 8 Done Sent Done false
156512023-09-22T23:31:08.136Z 19 Acked 1018 Write 4 Done Sent Done false
156522023-09-22T23:31:08.139Z 20 Acked 1019 Flush 0 Done Sent Done false
156532023-09-22T23:31:08.142Z 21 Acked 1020 Write 2 Done Sent Done false
156542023-09-22T23:31:08.145Z 22 Acked 1021 Read 10 Done Sent Done false
156552023-09-22T23:31:08.149Z 23 Acked 1022 Flush 0 Done Sent Done false
156562023-09-22T23:31:08.152Z 24 Acked 1023 Write 2 Done Sent Done false
156572023-09-22T23:31:08.156Z 25 Acked 1024 Read 7 Done Sent Done false
156582023-09-22T23:31:08.159Z 26 Acked 1025 Read 8 Done Sent Done false
156592023-09-22T23:31:08.162Z 27 Acked 1026 Write 4 Done Sent Done false
156602023-09-22T23:31:08.165Z 28 Acked 1027 Write 9 Done Sent Done false
156612023-09-22T23:31:08.168Z 29 Acked 1028 Read 3 Done Sent Done false
156622023-09-22T23:31:08.171Z 30 Acked 1029 Write 3 Sent Sent Sent false
156632023-09-22T23:31:08.174Z STATES DS:0 DS:1 DS:2 TOTAL
156642023-09-22T23:31:08.176Z New 0 0 0 0
156652023-09-22T23:31:08.179Z Sent 1 26 1 28
156662023-09-22T23:31:08.182Z Done 27 2 27 56
156672023-09-22T23:31:08.185Z Skipped 0 0 0 0
156682023-09-22T23:31:08.188Z Error 0 0 0 0
156692023-09-22T23:31:08.191ZLast Flush: 1022 1001 1022
156702023-09-22T23:31:08.194ZDownstairs last five completed: 1001 1000
156712023-09-22T23:31:08.196ZUpstairs last five completed: 30 29 28 27 26 25
156722023-09-22T23:31:08.199ZWrote out file "/tmp/test_repair_verify.data"
156732023-09-22T23:31:08.202Z
156742023-09-22T23:31:08.205ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
156752023-09-22T23:31:08.208Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.994156361Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1650}
156762023-09-22T23:31:08.210Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.994431479Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1650}
156772023-09-22T23:31:08.214Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:07.994455271Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1650}
156782023-09-22T23:31:08.216Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:08.020886172Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1650}
156792023-09-22T23:31:08.220Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:08.020920685Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1650}
156802023-09-22T23:31:08.225Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:08.02094027Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1650}
156812023-09-22T23:31:08.228Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:08.048473128Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1650}
156822023-09-22T23:31:08.231Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:08.04853078Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1650}
156832023-09-22T23:31:08.234Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:08.048570894Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1650}
156842023-09-22T23:31:08.237ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
156852023-09-22T23:31:08.240Z 0 000-019 118 117 118 177 174 177 F F F <---
156862023-09-22T23:31:08.243Z 1 020-039 118 116 118 176 173 176 F F F <---
156872023-09-22T23:31:08.246Z 2 040-059 116 116 116 173 173 173 F F F
156882023-09-22T23:31:08.249Z 3 060-079 118 118 118 175 175 175 F F F
156892023-09-22T23:31:08.252Z 4 080-099 118 118 118 175 175 175 F F F
156902023-09-22T23:31:08.255Z 5 100-119 113 113 113 170 170 170 F F F
156912023-09-22T23:31:08.258Z 6 120-139 115 115 115 172 172 172 F F F
156922023-09-22T23:31:08.260Z 7 140-159 116 116 116 173 173 173 F F F
156932023-09-22T23:31:08.263Z 8 160-179 114 114 114 171 171 171 F F F
156942023-09-22T23:31:08.266ZError: Difference in extent metadata found!
156952023-09-22T23:31:08.269Z 9 180-199 118 116 118 178 173 178 T F T <---
156962023-09-22T23:31:08.272Z 10 200-219 117 117 117 174 174 174 T F T <---
156972023-09-22T23:31:08.275Z 11 220-239 116 116 116 173 173 173 T F T <---
156982023-09-22T23:31:08.278Z 12 240-259 114 114 114 171 171 171 F F F
156992023-09-22T23:31:08.281Z 13 260-279 118 116 118 177 173 177 F F F <---
157002023-09-22T23:31:08.284Z 14 280-299 117 117 117 174 174 174 F F F
157012023-09-22T23:31:08.287Z 15 300-319 118 110 118 176 166 176 F T F <---
157022023-09-22T23:31:08.290Z 16 320-339 118 116 118 176 173 176 T T T <---
157032023-09-22T23:31:08.293Z 17 340-359 113 113 113 170 170 170 T F T <---
157042023-09-22T23:31:08.297Z 18 360-379 110 110 110 166 166 166 F F F
157052023-09-22T23:31:08.300Z 19 380-399 118 117 118 176 174 176 F F F <---
157062023-09-22T23:31:08.303Z 20 400-419 118 110 118 176 167 176 F F F <---
157072023-09-22T23:31:08.306Z 21 420-439 114 114 114 171 171 171 F F F
157082023-09-22T23:31:08.309Z 22 440-459 114 114 114 171 171 171 F F F
157092023-09-22T23:31:08.313Z 23 460-479 117 117 117 174 174 174 F F F
157102023-09-22T23:31:08.316Z 24 480-499 118 114 118 177 171 177 F F F <---
157112023-09-22T23:31:08.319Z 25 500-519 114 114 114 171 171 171 F F F
157122023-09-22T23:31:08.322Z 26 520-539 118 115 118 177 172 177 F F F <---
157132023-09-22T23:31:08.324Z 27 540-559 118 116 118 177 173 177 F F F <---
157142023-09-22T23:31:08.327Z 28 560-579 115 115 115 172 172 172 F F F
157152023-09-22T23:31:08.330Z 29 580-599 108 108 108 165 165 165 F F F
157162023-09-22T23:31:08.333ZMax gen: 118, Max flush: 178
157172023-09-22T23:31:08.335ZA Difference in extent metadata is expected here
157182023-09-22T23:31:08.338ZOn loop 59
157192023-09-22T23:31:08.341Z
157202023-09-22T23:31:08.344ZVerifying data now
157212023-09-22T23:31:14.558ZNo change after verify
157222023-09-22T23:31:14.561ZLoop: 59 Downstairs dump after verify (and repair):
157232023-09-22T23:31:14.576Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.576393081Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1655}
157242023-09-22T23:31:14.579Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.576707615Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1655}
157252023-09-22T23:31:14.582Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.576732411Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1655}
157262023-09-22T23:31:14.595Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.595666615Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1655}
157272023-09-22T23:31:14.598Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.59571932Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1655}
157282023-09-22T23:31:14.601Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.595740415Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1655}
157292023-09-22T23:31:14.606Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.606951577Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1655}
157302023-09-22T23:31:14.609Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.607003141Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1655}
157312023-09-22T23:31:14.612Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.607023461Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1655}
157322023-09-22T23:31:14.617ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
157332023-09-22T23:31:14.620Z 0 000-019 118 118 118 177 177 177 F F F
157342023-09-22T23:31:14.623Z 1 020-039 118 118 118 176 176 176 F F F
157352023-09-22T23:31:14.626Z 2 040-059 116 116 116 173 173 173 F F F
157362023-09-22T23:31:14.629Z 3 060-079 118 118 118 175 175 175 F F F
157372023-09-22T23:31:14.632Z 4 080-099 118 118 118 175 175 175 F F F
157382023-09-22T23:31:14.635Z 5 100-119 113 113 113 170 170 170 F F F
157392023-09-22T23:31:14.638Z 6 120-139 115 115 115 172 172 172 F F F
157402023-09-22T23:31:14.642Z 7 140-159 116 116 116 173 173 173 F F F
157412023-09-22T23:31:14.645Z 8 160-179 114 114 114 171 171 171 F F F
157422023-09-22T23:31:14.648Z 9 180-199 119 119 119 179 179 179 F F F
157432023-09-22T23:31:14.651Z 10 200-219 119 119 119 179 179 179 F F F
157442023-09-22T23:31:14.674Z 11 220-239 119 119 119 179 179 179 F F F
157452023-09-22T23:31:14.677Z 12 240-259 114 114 114 171 171 171 F F F
157462023-09-22T23:31:14.681Z 13 260-279 118 118 118 177 177 177 F F F
157472023-09-22T23:31:14.686Z 14 280-299 117 117 117 174 174 174 F F F
157482023-09-22T23:31:14.689Z 15 300-319 118 118 118 176 176 176 F F F
157492023-09-22T23:31:14.692Z 16 320-339 119 119 119 179 179 179 F F F
157502023-09-22T23:31:14.695Z 17 340-359 119 119 119 179 179 179 F F F
157512023-09-22T23:31:14.698Z 18 360-379 110 110 110 166 166 166 F F F
157522023-09-22T23:31:14.701Z 19 380-399 118 118 118 176 176 176 F F F
157532023-09-22T23:31:14.704Z 20 400-419 118 118 118 176 176 176 F F F
157542023-09-22T23:31:14.707Z 21 420-439 114 114 114 171 171 171 F F F
157552023-09-22T23:31:14.710Z 22 440-459 114 114 114 171 171 171 F F F
157562023-09-22T23:31:14.713Z 23 460-479 117 117 117 174 174 174 F F F
157572023-09-22T23:31:14.716Z 24 480-499 118 118 118 177 177 177 F F F
157582023-09-22T23:31:14.719Z 25 500-519 114 114 114 171 171 171 F F F
157592023-09-22T23:31:14.722Z 26 520-539 118 118 118 177 177 177 F F F
157602023-09-22T23:31:14.726Z 27 540-559 118 118 118 177 177 177 F F F
157612023-09-22T23:31:14.729Z 28 560-579 115 115 115 172 172 172 F F F
157622023-09-22T23:31:14.732Z 29 580-599 108 108 108 165 165 165 F F F
157632023-09-22T23:31:14.735ZMax gen: 119, Max flush: 179
157642023-09-22T23:31:14.738Z
157652023-09-22T23:31:14.740ZBegin loop 60
157662023-09-22T23:31:14.743ZDownstairs to restart: 0
157672023-09-22T23:31:14.746Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.657088674Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
157682023-09-22T23:31:14.749Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.657488775Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
157692023-09-22T23:31:14.752Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.657559245Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
157702023-09-22T23:31:14.755Z{"msg":"Crucible stats registered with UUID: 3dde999a-f9f6-410d-82e7-1287b5eab310","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.657627439Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
157712023-09-22T23:31:14.758Z{"msg":"Crucible 3dde999a-f9f6-410d-82e7-1287b5eab310 has session id: 6cfec578-34dd-4bca-aa2c-b44f54ae9124","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.657693105Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
157722023-09-22T23:31:14.761ZCrucible runtime is spawned
157732023-09-22T23:31:14.764Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.657892075Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657,"task":"up_listen"}
157742023-09-22T23:31:14.768Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.657940199Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
157752023-09-22T23:31:14.771ZThe guest has requested activation with gen:120
157762023-09-22T23:31:14.775Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.658026454Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
157772023-09-22T23:31:14.778Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.658170851Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657,"looper":"1"}
157782023-09-22T23:31:14.781Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.658509231Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657,"looper":"2"}
157792023-09-22T23:31:14.784Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.658680928Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657,"looper":"0"}
157802023-09-22T23:31:14.787Z{"msg":"[2] 3dde999a-f9f6-410d-82e7-1287b5eab310 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.658729575Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657,"looper":"2"}
157812023-09-22T23:31:14.790Z{"msg":"[1] 3dde999a-f9f6-410d-82e7-1287b5eab310 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.658768324Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657,"looper":"1"}
157822023-09-22T23:31:14.793Z{"msg":"3dde999a-f9f6-410d-82e7-1287b5eab310 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.658857035Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
157832023-09-22T23:31:14.797Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.658923081Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
157842023-09-22T23:31:14.800Z{"msg":"Set desired generation to :120","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.658988282Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
157852023-09-22T23:31:14.803Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.659310587Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
157862023-09-22T23:31:14.807Z{"msg":"[2] 3dde999a-f9f6-410d-82e7-1287b5eab310 (29c0ac3f-89f8-426e-a119-1c0aad2f5299) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.659551036Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
157872023-09-22T23:31:14.810Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.659692519Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
157882023-09-22T23:31:14.814Z{"msg":"[2] client is_active_req TRUE, promote! session 29c0ac3f-89f8-426e-a119-1c0aad2f5299","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.659753789Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
157892023-09-22T23:31:14.817Z{"msg":"[1] 3dde999a-f9f6-410d-82e7-1287b5eab310 (29c0ac3f-89f8-426e-a119-1c0aad2f5299) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.659781427Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
157902023-09-22T23:31:14.820Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.659972818Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
157912023-09-22T23:31:14.823Z{"msg":"[1] client is_active_req TRUE, promote! session 29c0ac3f-89f8-426e-a119-1c0aad2f5299","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.660048708Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
157922023-09-22T23:31:14.826Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.660357668Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
157932023-09-22T23:31:14.829Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.660440476Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
157942023-09-22T23:31:14.832Z{"msg":"3dde999a-f9f6-410d-82e7-1287b5eab310 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.66055314Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
157952023-09-22T23:31:14.835Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.660630068Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
157962023-09-22T23:31:14.838Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.660772595Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
157972023-09-22T23:31:14.841Z{"msg":"3dde999a-f9f6-410d-82e7-1287b5eab310 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.660829546Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
157982023-09-22T23:31:14.844Z{"msg":"[2] 3dde999a-f9f6-410d-82e7-1287b5eab310 (29c0ac3f-89f8-426e-a119-1c0aad2f5299) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.662747816Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
157992023-09-22T23:31:14.848Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.662875335Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158002023-09-22T23:31:14.851Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:31:14.662976131Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158012023-09-22T23:31:14.854Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.663022496Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158022023-09-22T23:31:14.857Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.6630622Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158032023-09-22T23:31:14.860Z{"msg":"3dde999a-f9f6-410d-82e7-1287b5eab310 New WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.66311215Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158042023-09-22T23:31:14.863Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.663158317Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158052023-09-22T23:31:14.866Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.663178641Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158062023-09-22T23:31:14.869Z{"msg":"[1] 3dde999a-f9f6-410d-82e7-1287b5eab310 (29c0ac3f-89f8-426e-a119-1c0aad2f5299) New WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.663218504Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158072023-09-22T23:31:14.873Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.66325764Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158082023-09-22T23:31:14.876Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:31:14.663295934Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158092023-09-22T23:31:14.879Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.663334103Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158102023-09-22T23:31:14.882Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.663372582Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158112023-09-22T23:31:14.885Z{"msg":"3dde999a-f9f6-410d-82e7-1287b5eab310 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.663392916Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158122023-09-22T23:31:14.889Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.663410957Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158132023-09-22T23:31:14.892Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:14.663428306Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158142023-09-22T23:31:15.660Z{"msg":"[0] 3dde999a-f9f6-410d-82e7-1287b5eab310 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.66022922Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657,"looper":"0"}
158152023-09-22T23:31:15.663Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.660311515Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158162023-09-22T23:31:15.666Z{"msg":"[0] 3dde999a-f9f6-410d-82e7-1287b5eab310 (29c0ac3f-89f8-426e-a119-1c0aad2f5299) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.661488666Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158172023-09-22T23:31:15.669Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.661544939Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158182023-09-22T23:31:15.673Z{"msg":"[0] client is_active_req TRUE, promote! session 29c0ac3f-89f8-426e-a119-1c0aad2f5299","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.661569025Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158192023-09-22T23:31:15.695Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.662199135Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158202023-09-22T23:31:15.698Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.662255189Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158212023-09-22T23:31:15.701Z{"msg":"3dde999a-f9f6-410d-82e7-1287b5eab310 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.662297833Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158222023-09-22T23:31:15.705Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.664449751Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158232023-09-22T23:31:15.708Z{"msg":"[0] 3dde999a-f9f6-410d-82e7-1287b5eab310 (29c0ac3f-89f8-426e-a119-1c0aad2f5299) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.664649224Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158242023-09-22T23:31:15.711Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.664732777Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158252023-09-22T23:31:15.715Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:31:15.664754055Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158262023-09-22T23:31:15.718Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.664771808Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158272023-09-22T23:31:15.721Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.664809825Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158282023-09-22T23:31:15.724Z{"msg":"3dde999a-f9f6-410d-82e7-1287b5eab310 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.664848768Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158292023-09-22T23:31:15.727Z{"msg":"[0]R flush_numbers[0..12]: [177, 176, 173, 175, 175, 170, 172, 173, 171, 179, 179, 179]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.664888357Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158302023-09-22T23:31:15.730Z{"msg":"[0]R generation[0..12]: [118, 118, 116, 118, 118, 113, 115, 116, 114, 119, 119, 119]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.664927562Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158312023-09-22T23:31:15.733Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.66496817Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158322023-09-22T23:31:15.736Z{"msg":"[1]R flush_numbers[0..12]: [177, 176, 173, 175, 175, 170, 172, 173, 171, 179, 179, 179]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.665040095Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158332023-09-22T23:31:15.739Z{"msg":"[1]R generation[0..12]: [118, 118, 116, 118, 118, 113, 115, 116, 114, 119, 119, 119]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.665064094Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158342023-09-22T23:31:15.743Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:120
158352023-09-22T23:31:15.746ZWait for a query_work_queue command to finish before sending IO
158362023-09-22T23:31:15.749Z,"time":"2023-09-22T23:31:15.665102815Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158372023-09-22T23:31:15.752Z{"msg":"[2]R flush_numbers[0..12]: [177, 176, 173, 175, 175, 170, 172, 173, 171, 179, 179, 179]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.665280986Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158382023-09-22T23:31:15.755Z{"msg":"[2]R generation[0..12]: [118, 118, 116, 118, 118, 113, 115, 116, 114, 119, 119, 119]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.665339559Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158392023-09-22T23:31:15.758Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.665403235Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158402023-09-22T23:31:15.761Z{"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.665446857Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158412023-09-22T23:31:15.764Z{"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.665506882Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158422023-09-22T23:31:15.767Z{"msg":"Next flush: 180","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.665556917Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158432023-09-22T23:31:15.769Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.665612154Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158442023-09-22T23:31:15.773Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.665666339Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158452023-09-22T23:31:15.775Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.665729461Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158462023-09-22T23:31:15.779Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.665788944Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158472023-09-22T23:31:15.782Z{"msg":"3dde999a-f9f6-410d-82e7-1287b5eab310 is now active with session: 29c0ac3f-89f8-426e-a119-1c0aad2f5299","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
158482023-09-22T23:31:15.785Z,"time":"2023-09-22T23:31:15.665859049Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158492023-09-22T23:31:15.788Z{"msg":"3dde999a-f9f6-410d-82e7-1287b5eab310 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.665968781Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158502023-09-22T23:31:15.792Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.666015477Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158512023-09-22T23:31:15.795Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.666053667Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158522023-09-22T23:31:15.798Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.666091571Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158532023-09-22T23:31:15.801Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.666129168Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158542023-09-22T23:31:15.807Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.666166504Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158552023-09-22T23:31:15.811Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.666204417Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158562023-09-22T23:31:15.814Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.666243116Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158572023-09-22T23:31:15.817Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.666280644Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158582023-09-22T23:31:15.820Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.666318043Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158592023-09-22T23:31:15.823Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.666355373Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158602023-09-22T23:31:15.826Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.666392724Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1657}
158612023-09-22T23:31:15.830ZLoading write count information from file "/tmp/test_repair_verify.data"
158622023-09-22T23:31:15.833ZRun Repair workload
158632023-09-22T23:31:15.835Z01/30 Flush
158642023-09-22T23:31:15.839Z02/30 Flush
158652023-09-22T23:31:15.842Z03/30 Write block 356 len 28672 data: 7 6 8 10 11 8 7
158662023-09-22T23:31:15.845Z04/30 Write block 457 len 24576 data: 8 8 12 11 11 9
158672023-09-22T23:31:15.847Z05/30 Write block 354 len 32768 data: 9 10 8 7 9 11 12 9
158682023-09-22T23:31:15.850Z06/30 Read block 455 len 24576
158692023-09-22T23:31:15.853Z07/30 Read block 218 len 4096
158702023-09-22T23:31:15.856Z08/30 Write block 98 len 36864 data: 13 11 10 10 11 12 14 16 14
158712023-09-22T23:31:15.859Z09/30 Flush
158722023-09-22T23:31:15.863Z10/30 Write block 174 len 32768 data: 12 14 16 15 15 11 9 10
158732023-09-22T23:31:15.866Z11/30 Read block 229 len 12288
158742023-09-22T23:31:15.869Z12/30 Write block 368 len 20480 data: 9 7 5 6 7
158752023-09-22T23:31:15.872Z13/30 Read block 7 len 20480
158762023-09-22T23:31:15.875Z14/30 Write block 482 len 4096 data: 9
158772023-09-22T23:31:15.877Z15/30 Read block 509 len 24576
158782023-09-22T23:31:15.881Z16/30 Read block 505 len 24576
158792023-09-22T23:31:15.884Z17/30 Read block 179 len 8192
158802023-09-22T23:31:15.887Z18/30 Write block 315 len 24576 data: 7 8 9 8 8 10
158812023-09-22T23:31:15.890Z19/30 Write block 57 len 28672 data: 4 8 9 12 12 14 17
158822023-09-22T23:31:15.893Z20/30 Read block 219 len 8192
158832023-09-22T23:31:15.896Z21/30 Read block 227 len 32768
158842023-09-22T23:31:15.900Z22/30 Write block 505 len 40960 data: 9 11 10 7 11 9 8 11 13 15
158852023-09-22T23:31:15.902Z23/30 Read block 262 len 28672
158862023-09-22T23:31:15.906Z24/30 Write block 154 len 12288 data: 8 9 8
158872023-09-22T23:31:15.909Z25/30 Read block 228 len 24576
158882023-09-22T23:31:15.912Z26/30 Write block 390 len 28672 data: 11 9 10 8 10 12 11
158892023-09-22T23:31:15.915Z27/30 Write block 16 len 4096 data: 9
158902023-09-22T23:31:15.918Z28/30 Read block 261 len 16384
158912023-09-22T23:31:15.921Z29/30 Read block 293 len 20480
158922023-09-22T23:31:15.924Z30/30 Write block 282 len 16384 data: 14 12 11 10
158932023-09-22T23:31:15.927Z----------------------------------------------------------------
158942023-09-22T23:31:15.931Z Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:28
158952023-09-22T23:31:15.935ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
158962023-09-22T23:31:15.938Z 3 Acked 1002 Write 7 Sent Done Done false
158972023-09-22T23:31:15.940Z 4 Acked 1003 Write 6 Sent Done Done false
158982023-09-22T23:31:15.944Z 5 Acked 1004 Write 8 Sent Done Done false
158992023-09-22T23:31:15.947Z 6 Acked 1005 Read 6 Sent Done Done false
159002023-09-22T23:31:15.950Z 7 Acked 1006 Read 1 Sent Done Done false
159012023-09-22T23:31:15.953Z 8 Acked 1007 Write 9 Sent Done Done false
159022023-09-22T23:31:15.956Z 9 Acked 1008 Flush 0 Sent Done Done false
159032023-09-22T23:31:15.958Z 10 Acked 1009 Write 8 Sent Done Done false
159042023-09-22T23:31:15.961Z 11 Acked 1010 Read 3 Sent Done Done false
159052023-09-22T23:31:15.964Z 12 Acked 1011 Write 5 Sent Done Done false
159062023-09-22T23:31:15.968Z 13 Acked 1012 Read 5 Sent Done Done false
159072023-09-22T23:31:15.971Z 14 Acked 1013 Write 1 Sent Done Done false
159082023-09-22T23:31:15.974Z 15 Acked 1014 Read 6 Sent Done Done false
159092023-09-22T23:31:15.977Z 16 Acked 1015 Read 6 Sent Done Done false
159102023-09-22T23:31:15.981Z 17 Acked 1016 Read 2 Sent Done Done false
159112023-09-22T23:31:15.983Z 18 Acked 1017 Write 6 Sent Done Done false
159122023-09-22T23:31:15.987Z 19 Acked 1018 Write 7 Sent Done Done false
159132023-09-22T23:31:15.989Z 20 Acked 1019 Read 2 Sent Done Done false
159142023-09-22T23:31:15.992Z 21 Acked 1020 Read 8 Sent Done Done false
159152023-09-22T23:31:15.996Z 22 Acked 1021 Write 10 Sent Done Done false
159162023-09-22T23:31:15.998Z 23 Acked 1022 Read 7 Sent Done Done false
159172023-09-22T23:31:16.002Z 24 Acked 1023 Write 3 Sent Done Done false
159182023-09-22T23:31:16.005Z 25 Acked 1024 Read 6 Sent Done Done false
159192023-09-22T23:31:16.008Z 26 Acked 1025 Write 7 Sent Done Done false
159202023-09-22T23:31:16.012Z 27 Acked 1026 Write 1 Sent Done Done false
159212023-09-22T23:31:16.014Z 28 Acked 1027 Read 4 Sent Done Done false
159222023-09-22T23:31:16.017Z 29 Acked 1028 Read 5 Sent Done Done false
159232023-09-22T23:31:16.020Z 30 Acked 1029 Write 4 Sent Sent Sent false
159242023-09-22T23:31:16.023Z STATES DS:0 DS:1 DS:2 TOTAL
159252023-09-22T23:31:16.026Z New 0 0 0 0
159262023-09-22T23:31:16.029Z Sent 28 1 1 30
159272023-09-22T23:31:16.032Z Done 0 27 27 54
159282023-09-22T23:31:16.035Z Skipped 0 0 0 0
159292023-09-22T23:31:16.038Z Error 0 0 0 0
159302023-09-22T23:31:16.041ZLast Flush: 1001 1008 1008
159312023-09-22T23:31:16.044ZDownstairs last five completed: 1001 1000
159322023-09-22T23:31:16.048ZUpstairs last five completed: 30 29 28 27 26 25
159332023-09-22T23:31:16.051ZWrote out file "/tmp/test_repair_verify.data"
159342023-09-22T23:31:16.054Z
159352023-09-22T23:31:16.057ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
159362023-09-22T23:31:16.060Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.810822884Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1658}
159372023-09-22T23:31:16.062Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.81117936Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1658}
159382023-09-22T23:31:16.065Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.811242829Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1658}
159392023-09-22T23:31:16.068Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.839150967Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1658}
159402023-09-22T23:31:16.071Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.839183573Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1658}
159412023-09-22T23:31:16.074Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.839217364Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1658}
159422023-09-22T23:31:16.077Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.869945223Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1658}
159432023-09-22T23:31:16.080Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.869977213Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1658}
159442023-09-22T23:31:16.084Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:15.869996295Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1658}
159452023-09-22T23:31:16.087ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
159462023-09-22T23:31:16.090Z 0 000-019 118 118 118 177 177 177 F T T <---
159472023-09-22T23:31:16.093Z 1 020-039 118 118 118 176 176 176 F F F
159482023-09-22T23:31:16.095Z 2 040-059 116 116 116 173 173 173 F T T <---
159492023-09-22T23:31:16.098ZError: Difference in extent metadata found!
159502023-09-22T23:31:16.101Z 3 060-079 118 118 118 175 175 175 F T T <---
159512023-09-22T23:31:16.104Z 4 080-099 118 120 120 175 182 182 F F F <---
159522023-09-22T23:31:16.107Z 5 100-119 113 120 120 170 182 182 F F F <---
159532023-09-22T23:31:16.110Z 6 120-139 115 115 115 172 172 172 F F F
159542023-09-22T23:31:16.113Z 7 140-159 116 116 116 173 173 173 F T T <---
159552023-09-22T23:31:16.116Z 8 160-179 114 114 114 171 171 171 F T T <---
159562023-09-22T23:31:16.118Z 9 180-199 119 119 119 179 179 179 F T T <---
159572023-09-22T23:31:16.121Z 10 200-219 119 119 119 179 179 179 F F F
159582023-09-22T23:31:16.124Z 11 220-239 119 119 119 179 179 179 F F F
159592023-09-22T23:31:16.127Z 12 240-259 114 114 114 171 171 171 F F F
159602023-09-22T23:31:16.131Z 13 260-279 118 118 118 177 177 177 F F F
159612023-09-22T23:31:16.134Z 14 280-299 117 117 117 174 174 174 F T T <---
159622023-09-22T23:31:16.136Z 15 300-319 118 118 118 176 176 176 F T T <---
159632023-09-22T23:31:16.139Z 16 320-339 119 119 119 179 179 179 F T T <---
159642023-09-22T23:31:16.142Z 17 340-359 119 120 120 179 182 182 F F F <---
159652023-09-22T23:31:16.145Z 18 360-379 110 120 120 166 182 182 F T T <---
159662023-09-22T23:31:16.148Z 19 380-399 118 118 118 176 176 176 F T T <---
159672023-09-22T23:31:16.152Z 20 400-419 118 118 118 176 176 176 F F F
159682023-09-22T23:31:16.155Z 21 420-439 114 114 114 171 171 171 F F F
159692023-09-22T23:31:16.158Z 22 440-459 114 120 120 171 182 182 F F F <---
159702023-09-22T23:31:16.161Z 23 460-479 117 120 120 174 182 182 F F F <---
159712023-09-22T23:31:16.165Z 24 480-499 118 118 118 177 177 177 F T T <---
159722023-09-22T23:31:16.168Z 25 500-519 114 114 114 171 171 171 F T T <---
159732023-09-22T23:31:16.171Z 26 520-539 118 118 118 177 177 177 F F F
159742023-09-22T23:31:16.174Z 27 540-559 118 118 118 177 177 177 F F F
159752023-09-22T23:31:16.178Z 28 560-579 115 115 115 172 172 172 F F F
159762023-09-22T23:31:16.182Z 29 580-599 108 108 108 165 165 165 F F F
159772023-09-22T23:31:16.185ZMax gen: 120, Max flush: 182
159782023-09-22T23:31:16.188ZA Difference in extent metadata is expected here
159792023-09-22T23:31:16.193ZOn loop 60
159802023-09-22T23:31:16.196Z
159812023-09-22T23:31:16.199ZVerifying data now
159822023-09-22T23:31:22.751ZNo change after verify
159832023-09-22T23:31:22.755ZLoop: 60 Downstairs dump after verify (and repair):
159842023-09-22T23:31:22.772Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.770136958Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1663}
159852023-09-22T23:31:22.780Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.770574722Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1663}
159862023-09-22T23:31:22.793Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.770599243Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1663}
159872023-09-22T23:31:22.803Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.789758191Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1663}
159882023-09-22T23:31:22.816Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.789813858Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1663}
159892023-09-22T23:31:23.240Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.789851908Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1663}
159902023-09-22T23:31:23.249Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.801042053Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1663}
159912023-09-22T23:31:23.264Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.801093077Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1663}
159922023-09-22T23:31:23.271Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.801130642Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1663}
159932023-09-22T23:31:23.274ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
159942023-09-22T23:31:23.279Z 0 000-019 121 121 121 183 183 183 F F F
159952023-09-22T23:31:23.283Z 1 020-039 118 118 118 176 176 176 F F F
159962023-09-22T23:31:23.286Z 2 040-059 121 121 121 183 183 183 F F F
159972023-09-22T23:31:23.289Z 3 060-079 121 121 121 183 183 183 F F F
159982023-09-22T23:31:23.291Z 4 080-099 120 120 120 182 182 182 F F F
159992023-09-22T23:31:23.294Z 5 100-119 120 120 120 182 182 182 F F F
160002023-09-22T23:31:23.297Z 6 120-139 115 115 115 172 172 172 F F F
160012023-09-22T23:31:23.300Z 7 140-159 121 121 121 183 183 183 F F F
160022023-09-22T23:31:23.303Z 8 160-179 121 121 121 183 183 183 F F F
160032023-09-22T23:31:23.305Z 9 180-199 121 121 121 183 183 183 F F F
160042023-09-22T23:31:23.308Z 10 200-219 119 119 119 179 179 179 F F F
160052023-09-22T23:31:23.311Z 11 220-239 119 119 119 179 179 179 F F F
160062023-09-22T23:31:23.314Z 12 240-259 114 114 114 171 171 171 F F F
160072023-09-22T23:31:23.317Z 13 260-279 118 118 118 177 177 177 F F F
160082023-09-22T23:31:23.320Z 14 280-299 121 121 121 183 183 183 F F F
160092023-09-22T23:31:23.323Z 15 300-319 121 121 121 183 183 183 F F F
160102023-09-22T23:31:23.326Z 16 320-339 121 121 121 183 183 183 F F F
160112023-09-22T23:31:23.329Z 17 340-359 120 120 120 182 182 182 F F F
160122023-09-22T23:31:23.332Z 18 360-379 121 121 121 183 183 183 F F F
160132023-09-22T23:31:23.335Z 19 380-399 121 121 121 183 183 183 F F F
160142023-09-22T23:31:23.338Z 20 400-419 118 118 118 176 176 176 F F F
160152023-09-22T23:31:23.341Z 21 420-439 114 114 114 171 171 171 F F F
160162023-09-22T23:31:23.344Z 22 440-459 120 120 120 182 182 182 F F F
160172023-09-22T23:31:23.347Z 23 460-479 120 120 120 182 182 182 F F F
160182023-09-22T23:31:23.350Z 24 480-499 121 121 121 183 183 183 F F F
160192023-09-22T23:31:23.353Z 25 500-519 121 121 121 183 183 183 F F F
160202023-09-22T23:31:23.357Z 26 520-539 118 118 118 177 177 177 F F F
160212023-09-22T23:31:23.359Z 27 540-559 118 118 118 177 177 177 F F F
160222023-09-22T23:31:23.362Z 28 560-579 115 115 115 172 172 172 F F F
160232023-09-22T23:31:23.365Z 29 580-599 108 108 108 165 165 165 F F F
160242023-09-22T23:31:23.368ZMax gen: 121, Max flush: 183
160252023-09-22T23:31:23.371Z
160262023-09-22T23:31:23.374ZBegin loop 61
160272023-09-22T23:31:23.377ZDownstairs to restart: 0
160282023-09-22T23:31:23.380Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.85031367Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160292023-09-22T23:31:23.383Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.850682675Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160302023-09-22T23:31:23.386Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.850743585Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160312023-09-22T23:31:23.389Z{"msg":"Crucible stats registered with UUID: f832148f-24df-418d-b670-433a8291b420","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.850799068Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160322023-09-22T23:31:23.392Z{"msg":"Crucible f832148f-24df-418d-b670-433a8291b420 has session id: 63539074-6d00-48d0-a7b5-3c465859d41e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.850840923Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160332023-09-22T23:31:23.395ZCrucible runtime is spawned
160342023-09-22T23:31:23.398Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30The guest has requested activation with gen:122
160352023-09-22T23:31:23.401Z,"time":"2023-09-22T23:31:22.851085578Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665,"task":"up_listen"}
160362023-09-22T23:31:23.404Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.851199988Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160372023-09-22T23:31:23.407Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.851240011Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160382023-09-22T23:31:23.409Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.851274415Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665,"looper":"2"}
160392023-09-22T23:31:23.412Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.851622519Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665,"looper":"0"}
160402023-09-22T23:31:23.416Z{"msg":"[2] f832148f-24df-418d-b670-433a8291b420 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.85173533Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665,"looper":"2"}
160412023-09-22T23:31:23.418Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.851795438Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665,"looper":"1"}
160422023-09-22T23:31:23.421Z{"msg":"[1] f832148f-24df-418d-b670-433a8291b420 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.851953158Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665,"looper":"1"}
160432023-09-22T23:31:23.424Z{"msg":"f832148f-24df-418d-b670-433a8291b420 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.852226775Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160442023-09-22T23:31:23.427Z{"msg":"Set desired generation to :122","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.852283798Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160452023-09-22T23:31:23.430Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.852324241Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160462023-09-22T23:31:23.433Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.852650255Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160472023-09-22T23:31:23.436Z{"msg":"[2] f832148f-24df-418d-b670-433a8291b420 (c159ee0f-860f-4a79-9cb4-f9d2939ef21b) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.852979927Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160482023-09-22T23:31:23.439Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.853039614Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160492023-09-22T23:31:23.442Z{"msg":"[2] client is_active_req TRUE, promote! session c159ee0f-860f-4a79-9cb4-f9d2939ef21b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.853062338Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160502023-09-22T23:31:23.445Z{"msg":"[1] f832148f-24df-418d-b670-433a8291b420 (c159ee0f-860f-4a79-9cb4-f9d2939ef21b) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.853170304Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160512023-09-22T23:31:23.448Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.853244229Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160522023-09-22T23:31:23.451Z{"msg":"[1] client is_active_req TRUE, promote! session c159ee0f-860f-4a79-9cb4-f9d2939ef21b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.853300379Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160532023-09-22T23:31:23.454Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.853677344Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160542023-09-22T23:31:23.457Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.853755846Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160552023-09-22T23:31:23.460Z{"msg":"f832148f-24df-418d-b670-433a8291b420 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.853827724Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160562023-09-22T23:31:23.462Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.853869954Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160572023-09-22T23:31:23.465Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.853915449Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160582023-09-22T23:31:23.468Z{"msg":"f832148f-24df-418d-b670-433a8291b420 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.853954704Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160592023-09-22T23:31:23.471Z{"msg":"[1] f832148f-24df-418d-b670-433a8291b420 (c159ee0f-860f-4a79-9cb4-f9d2939ef21b) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.856223783Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160602023-09-22T23:31:23.474Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.856272774Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160612023-09-22T23:31:23.477Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:31:22.856310871Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160622023-09-22T23:31:23.480Z{"msg":"[2] f832148f-24df-418d-b670-433a8291b420 (c159ee0f-860f-4a79-9cb4-f9d2939ef21b) New WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.856408493Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160632023-09-22T23:31:23.483Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.856443109Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160642023-09-22T23:31:23.486Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.856462213Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160652023-09-22T23:31:23.490Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:31:22.856479609Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160662023-09-22T23:31:23.493Z{"msg":"f832148f-24df-418d-b670-433a8291b420 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.856498186Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160672023-09-22T23:31:23.497Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.856516013Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160682023-09-22T23:31:23.500Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.856611819Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160692023-09-22T23:31:23.503Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.856637848Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160702023-09-22T23:31:23.507Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.856655925Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160712023-09-22T23:31:23.510Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.856674455Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160722023-09-22T23:31:23.513Z{"msg":"f832148f-24df-418d-b670-433a8291b420 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.856692991Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160732023-09-22T23:31:23.516Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.85671069Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160742023-09-22T23:31:23.519Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:22.856728237Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160752023-09-22T23:31:23.852Z{"msg":"[0] f832148f-24df-418d-b670-433a8291b420 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.852721334Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665,"looper":"0"}
160762023-09-22T23:31:23.876Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.852805816Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160772023-09-22T23:31:23.880Z{"msg":"[0] f832148f-24df-418d-b670-433a8291b420 (c159ee0f-860f-4a79-9cb4-f9d2939ef21b) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.853984127Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160782023-09-22T23:31:23.883Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.854043365Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160792023-09-22T23:31:23.886Z{"msg":"[0] client is_active_req TRUE, promote! session c159ee0f-860f-4a79-9cb4-f9d2939ef21b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.854069151Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160802023-09-22T23:31:23.889Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.854727552Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160812023-09-22T23:31:23.892Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.854790336Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160822023-09-22T23:31:23.895Z{"msg":"f832148f-24df-418d-b670-433a8291b420 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.854832837Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160832023-09-22T23:31:23.898Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.857136623Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160842023-09-22T23:31:23.901Z{"msg":"[0] f832148f-24df-418d-b670-433a8291b420 (c159ee0f-860f-4a79-9cb4-f9d2939ef21b) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.857274967Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160852023-09-22T23:31:23.904Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.857327597Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160862023-09-22T23:31:23.907Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:31:23.857350234Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160872023-09-22T23:31:23.910Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.857392671Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160882023-09-22T23:31:23.913Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.857416087Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160892023-09-22T23:31:23.916Z{"msg":"f832148f-24df-418d-b670-433a8291b420 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.857454088Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160902023-09-22T23:31:23.919Z{"msg":"[0]R flush_numbers[0..12]: [183, 176, 183, 183, 182, 182, 172, 183, 183, 183, 179, 179]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.85749516Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160912023-09-22T23:31:23.924Z{"msg":"[0]R generation[0..12]: [121, 118, 121, 121, 120, 120, 115, 121, 121, 121, 119, 119]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.857536587Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160922023-09-22T23:31:23.927Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.857577445Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160932023-09-22T23:31:23.931Z{"msg":"[1]R flush_numbers[0..12]: [183, 176, 183, 183, 182, 182, 172, 183, 183, 183, 179, 179]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.857617299Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160942023-09-22T23:31:23.934Z{"msg":"[1]R generation[0..12]: [121, 118, 121, 121, 120, 120, 115, 121, 121, 121, 119, 119]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.857660479Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160952023-09-22T23:31:23.936Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.857700627Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160962023-09-22T23:31:23.940Z{"msg":"[2]R flush_numbers[0..12]: [183, 176, 183, 183, 182, 182, 172, 183, 183, 183, 179, 179]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:122
160972023-09-22T23:31:23.945ZWait for a query_work_queue command to finish before sending IO
160982023-09-22T23:31:23.948Z,"time":"2023-09-22T23:31:23.857767383Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
160992023-09-22T23:31:23.951Z{"msg":"[2]R generation[0..12]: [121, 118, 121, 121, 120, 120, 115, 121, 121, 121, 119, 119]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.857954422Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
161002023-09-22T23:31:23.954Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.858022918Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
161012023-09-22T23:31:23.956Z{"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.858055339Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
161022023-09-22T23:31:23.959Z{"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.858103679Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
161032023-09-22T23:31:23.962Z{"msg":"Next flush: 184","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.858151556Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
161042023-09-22T23:31:23.965Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.85818934Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
161052023-09-22T23:31:23.968Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.858238042Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
161062023-09-22T23:31:23.971Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.858285629Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
161072023-09-22T23:31:23.974Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.858325001Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
161082023-09-22T23:31:23.977Z{"msg":"f832148f-24df-418d-b670-433a8291b420 is now active with session: c159ee0f-860f-4a79-9cb4-f9d2939ef21b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.858370715Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
161092023-09-22T23:31:23.980Z{"msg":"f832148f-24df-418d-b670-433a8291b420 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.858409749Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
161102023-09-22T23:31:23.983Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.858453191Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
161112023-09-22T23:31:23.986Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.858491399Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
161122023-09-22T23:31:23.989Z{Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
161132023-09-22T23:31:23.992Z"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.858535603Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
161142023-09-22T23:31:23.994Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.858574057Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
161152023-09-22T23:31:23.998Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.858611516Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
161162023-09-22T23:31:24.001Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.858649445Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
161172023-09-22T23:31:24.004Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.858686178Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
161182023-09-22T23:31:24.007Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.858723245Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
161192023-09-22T23:31:24.011Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.858760391Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
161202023-09-22T23:31:24.014Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.858797533Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
161212023-09-22T23:31:24.017Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:23.858834225Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1665}
161222023-09-22T23:31:24.020ZLoading write count information from file "/tmp/test_repair_verify.data"
161232023-09-22T23:31:24.023ZRun Repair workload
161242023-09-22T23:31:24.026Z01/30 Write block 187 len 32768 data: 10 8 8 10 10 8 8 10
161252023-09-22T23:31:24.029Z02/30 Read block 581 len 40960
161262023-09-22T23:31:24.032Z03/30 Write block 294 len 16384 data: 11 12 11 8
161272023-09-22T23:31:24.035Z04/30 Write block 535 len 8192 data: 11 11
161282023-09-22T23:31:24.038Z05/30 Write block 359 len 32768 data: 12 13 10 8 7 6 6 7
161292023-09-22T23:31:24.041Z06/30 Flush
161302023-09-22T23:31:24.044Z07/30 Write block 24 len 4096 data: 7
161312023-09-22T23:31:24.048Z08/30 Read block 115 len 40960
161322023-09-22T23:31:24.051Z09/30 Read block 114 len 36864
161332023-09-22T23:31:24.053Z10/30 Write block 40 len 12288 data: 11 10 9
161342023-09-22T23:31:24.056Z11/30 Read block 505 len 16384
161352023-09-22T23:31:24.059Z12/30 Read block 86 len 28672
161362023-09-22T23:31:24.063Z13/30 Write block 150 len 8192 data: 11 11
161372023-09-22T23:31:24.065Z14/30 Flush
161382023-09-22T23:31:24.068Z15/30 Write block 140 len 12288 data: 9 9 9
161392023-09-22T23:31:24.071Z16/30 Write block 127 len 28672 data: 9 9 7 8 7 11 10
161402023-09-22T23:31:24.074Z17/30 Read block 14 len 32768
161412023-09-22T23:31:24.077Z18/30 Read block 191 len 8192
161422023-09-22T23:31:24.080Z19/30 Write block 459 len 8192 data: 13 12
161432023-09-22T23:31:24.083Z20/30 Write block 173 len 20480 data: 12 13 15 17 16
161442023-09-22T23:31:24.086Z21/30 Read block 562 len 40960
161452023-09-22T23:31:24.089Z22/30 Write block 353 len 40960 data: 9 10 11 9 8 10 13 14 11 9
161462023-09-22T23:31:24.092Z23/30 Read block 396 len 16384
161472023-09-22T23:31:24.095Z24/30 Read block 85 len 8192
161482023-09-22T23:31:24.097Z25/30 Read block 84 len 36864
161492023-09-22T23:31:24.101Z26/30 Read block 385 len 28672
161502023-09-22T23:31:24.104Z27/30 Write block 520 len 16384 data: 11 9 9 9
161512023-09-22T23:31:24.106Z28/30 Read block 493 len 16384
161522023-09-22T23:31:24.109Z29/30 Read block 509 len 16384
161532023-09-22T23:31:24.113Z30/30 Read block 341 len 20480
161542023-09-22T23:31:24.115Z----------------------------------------------------------------
161552023-09-22T23:31:24.118Z Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:30
161562023-09-22T23:31:24.121ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
161572023-09-22T23:31:24.124Z 1 Acked 1000 Write 8 Done Done Done false
161582023-09-22T23:31:24.127Z 2 Acked 1001 Read 10 Done Done Done false
161592023-09-22T23:31:24.131Z 3 Acked 1002 Write 4 Sent Done Done false
161602023-09-22T23:31:24.137Z 4 Acked 1003 Write 2 Sent Done Done false
161612023-09-22T23:31:24.141Z 5 Acked 1004 Write 8 Sent Done Done false
161622023-09-22T23:31:24.144Z 6 Acked 1005 Flush 0 Sent Done Done false
161632023-09-22T23:31:24.146Z 7 Acked 1006 Write 1 Sent Done Done false
161642023-09-22T23:31:24.149Z 8 Acked 1007 Read 10 Sent Done Done false
161652023-09-22T23:31:24.152Z 9 Acked 1008 Read 9 Sent Done Done false
161662023-09-22T23:31:24.156Z 10 Acked 1009 Write 3 Sent Done Done false
161672023-09-22T23:31:24.159Z 11 Acked 1010 Read 4 Sent Done Done false
161682023-09-22T23:31:24.162Z 12 Acked 1011 Read 7 Sent Done Done false
161692023-09-22T23:31:24.165Z 13 Acked 1012 Write 2 Sent Done Done false
161702023-09-22T23:31:24.168Z 14 Acked 1013 Flush 0 Sent Done Done false
161712023-09-22T23:31:24.170Z 15 Acked 1014 Write 3 Sent Done Done false
161722023-09-22T23:31:24.174Z 16 Acked 1015 Write 7 Sent Done Done false
161732023-09-22T23:31:24.177Z 17 Acked 1016 Read 8 Sent Done Done false
161742023-09-22T23:31:24.180Z 18 Acked 1017 Read 2 Sent Done Done false
161752023-09-22T23:31:24.183Z 19 Acked 1018 Write 2 Sent Done Done false
161762023-09-22T23:31:24.186Z 20 Acked 1019 Write 5 Sent Done Done false
161772023-09-22T23:31:24.189Z 21 Acked 1020 Read 10 Sent Done Done false
161782023-09-22T23:31:24.192Z 22 Acked 1021 Write 10 Sent Done Done false
161792023-09-22T23:31:24.195Z 23 Acked 1022 Read 4 Sent Done Done false
161802023-09-22T23:31:24.198Z 24 Acked 1023 Read 2 Sent Done Done false
161812023-09-22T23:31:24.201Z 25 Acked 1024 Read 9 Sent Done Done false
161822023-09-22T23:31:24.204Z 26 Acked 1025 Read 7 Sent Done Done false
161832023-09-22T23:31:24.207Z 27 Acked 1026 Write 4 Sent Done Done false
161842023-09-22T23:31:24.209Z 28 Acked 1027 Read 4 Sent Done Done false
161852023-09-22T23:31:24.212Z 29 Acked 1028 Read 4 Sent Done Done false
161862023-09-22T23:31:24.215Z 30 Acked 1029 Read 5 Sent Sent Done false
161872023-09-22T23:31:24.218Z STATES DS:0 DS:1 DS:2 TOTAL
161882023-09-22T23:31:24.221Z New 0 0 0 0
161892023-09-22T23:31:24.223Z Sent 28 1 0 29
161902023-09-22T23:31:24.227Z Done 2 29 30 61
161912023-09-22T23:31:24.230Z Skipped 0 0 0 0
161922023-09-22T23:31:24.233Z Error 0 0 0 0
161932023-09-22T23:31:24.236ZLast Flush: 0 1013 1013
161942023-09-22T23:31:24.239ZDownstairs last five completed:
161952023-09-22T23:31:24.241ZUpstairs last five completed: 30 29 28 27 26 25
161962023-09-22T23:31:24.244ZWrote out file "/tmp/test_repair_verify.data"
161972023-09-22T23:31:24.247Z
161982023-09-22T23:31:24.249ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
161992023-09-22T23:31:24.252Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:24.004456782Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1666}
162002023-09-22T23:31:24.255Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:24.004782755Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1666}
162012023-09-22T23:31:24.258Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:24.004806411Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1666}
162022023-09-22T23:31:24.260Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:24.0331293Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1666}
162032023-09-22T23:31:24.263Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:24.033165141Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1666}
162042023-09-22T23:31:24.266Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:24.033185079Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1666}
162052023-09-22T23:31:24.270Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:24.056738009Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1666}
162062023-09-22T23:31:24.272Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:24.056770854Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1666}
162072023-09-22T23:31:24.275ZError: Difference in extent metadata found!
162082023-09-22T23:31:24.278Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:24.056790213Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1666}
162092023-09-22T23:31:24.281ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
162102023-09-22T23:31:24.284Z 0 000-019 121 121 121 183 183 183 F F F
162112023-09-22T23:31:24.287Z 1 020-039 118 122 122 176 185 185 F F F <---
162122023-09-22T23:31:24.290Z 2 040-059 121 122 122 183 185 185 F F F <---
162132023-09-22T23:31:24.293Z 3 060-079 121 121 121 183 183 183 F F F
162142023-09-22T23:31:24.295Z 4 080-099 120 120 120 182 182 182 F F F
162152023-09-22T23:31:24.298Z 5 100-119 120 120 120 182 182 182 F F F
162162023-09-22T23:31:24.301Z 6 120-139 115 115 115 172 172 172 F T T <---
162172023-09-22T23:31:24.303Z 7 140-159 121 122 122 183 185 185 F T T <---
162182023-09-22T23:31:24.306Z 8 160-179 121 121 121 183 183 183 F T T <---
162192023-09-22T23:31:24.309Z 9 180-199 121 122 122 183 184 184 T F F <---
162202023-09-22T23:31:24.312Z 10 200-219 119 119 119 179 179 179 F F F
162212023-09-22T23:31:24.315Z 11 220-239 119 119 119 179 179 179 F F F
162222023-09-22T23:31:24.317Z 12 240-259 114 114 114 171 171 171 F F F
162232023-09-22T23:31:24.321Z 13 260-279 118 118 118 177 177 177 F F F
162242023-09-22T23:31:24.324Z 14 280-299 121 122 122 183 184 184 F F F <---
162252023-09-22T23:31:24.326Z 15 300-319 121 121 121 183 183 183 F F F
162262023-09-22T23:31:24.329Z 16 320-339 121 121 121 183 183 183 F F F
162272023-09-22T23:31:24.332Z 17 340-359 120 122 122 182 184 184 F T T <---
162282023-09-22T23:31:24.335Z 18 360-379 121 122 122 183 184 184 F T T <---
162292023-09-22T23:31:24.337Z 19 380-399 121 121 121 183 183 183 F F F
162302023-09-22T23:31:24.340Z 20 400-419 118 118 118 176 176 176 F F F
162312023-09-22T23:31:24.343Z 21 420-439 114 114 114 171 171 171 F F F
162322023-09-22T23:31:24.346Z 22 440-459 120 120 120 182 182 182 F T T <---
162332023-09-22T23:31:24.349Z 23 460-479 120 120 120 182 182 182 F T T <---
162342023-09-22T23:31:24.351Z 24 480-499 121 121 121 183 183 183 F F F
162352023-09-22T23:31:24.354Z 25 500-519 121 121 121 183 183 183 F F F
162362023-09-22T23:31:24.357Z 26 520-539 118 122 122 177 184 184 F T T <---
162372023-09-22T23:31:24.360Z 27 540-559 118 118 118 177 177 177 F F F
162382023-09-22T23:31:24.363Z 28 560-579 115 115 115 172 172 172 F F F
162392023-09-22T23:31:24.366Z 29 580-599 108 108 108 165 165 165 F F F
162402023-09-22T23:31:24.368ZMax gen: 122, Max flush: 185
162412023-09-22T23:31:24.371ZA Difference in extent metadata is expected here
162422023-09-22T23:31:24.374ZOn loop 61
162432023-09-22T23:31:24.376Z
162442023-09-22T23:31:24.379ZVerifying data now
162452023-09-22T23:31:30.430ZNo change after verify
162462023-09-22T23:31:30.433ZLoop: 61 Downstairs dump after verify (and repair):
162472023-09-22T23:31:30.448Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.448473666Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1671}
162482023-09-22T23:31:30.451Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.448886039Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1671}
162492023-09-22T23:31:30.454Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.448911623Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1671}
162502023-09-22T23:31:30.467Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.4675393Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1671}
162512023-09-22T23:31:30.470Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.467595625Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1671}
162522023-09-22T23:31:30.472Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.467617042Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1671}
162532023-09-22T23:31:30.477Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.478424119Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1671}
162542023-09-22T23:31:30.481Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.478478843Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1671}
162552023-09-22T23:31:30.484Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.478499761Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1671}
162562023-09-22T23:31:30.488ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
162572023-09-22T23:31:30.492Z 0 000-019 121 121 121 183 183 183 F F F
162582023-09-22T23:31:30.494Z 1 020-039 122 122 122 185 185 185 F F F
162592023-09-22T23:31:30.497Z 2 040-059 122 122 122 185 185 185 F F F
162602023-09-22T23:31:30.500Z 3 060-079 121 121 121 183 183 183 F F F
162612023-09-22T23:31:30.503Z 4 080-099 120 120 120 182 182 182 F F F
162622023-09-22T23:31:30.506Z 5 100-119 120 120 120 182 182 182 F F F
162632023-09-22T23:31:30.509Z 6 120-139 123 123 123 186 186 186 F F F
162642023-09-22T23:31:30.513Z 7 140-159 123 123 123 186 186 186 F F F
162652023-09-22T23:31:30.516Z 8 160-179 123 123 123 186 186 186 F F F
162662023-09-22T23:31:30.519Z 9 180-199 122 122 122 184 184 184 F F F
162672023-09-22T23:31:30.521Z 10 200-219 119 119 119 179 179 179 F F F
162682023-09-22T23:31:30.524Z 11 220-239 119 119 119 179 179 179 F F F
162692023-09-22T23:31:30.527Z 12 240-259 114 114 114 171 171 171 F F F
162702023-09-22T23:31:30.530Z 13 260-279 118 118 118 177 177 177 F F F
162712023-09-22T23:31:30.533Z 14 280-299 122 122 122 184 184 184 F F F
162722023-09-22T23:31:30.536Z 15 300-319 121 121 121 183 183 183 F F F
162732023-09-22T23:31:30.539Z 16 320-339 121 121 121 183 183 183 F F F
162742023-09-22T23:31:30.542Z 17 340-359 123 123 123 186 186 186 F F F
162752023-09-22T23:31:30.544Z 18 360-379 123 123 123 186 186 186 F F F
162762023-09-22T23:31:30.547Z 19 380-399 121 121 121 183 183 183 F F F
162772023-09-22T23:31:30.550Z 20 400-419 118 118 118 176 176 176 F F F
162782023-09-22T23:31:30.553Z 21 420-439 114 114 114 171 171 171 F F F
162792023-09-22T23:31:30.556Z 22 440-459 123 123 123 186 186 186 F F F
162802023-09-22T23:31:30.559Z 23 460-479 123 123 123 186 186 186 F F F
162812023-09-22T23:31:30.562Z 24 480-499 121 121 121 183 183 183 F F F
162822023-09-22T23:31:30.565Z 25 500-519 121 121 121 183 183 183 F F F
162832023-09-22T23:31:30.568Z 26 520-539 123 123 123 186 186 186 F F F
162842023-09-22T23:31:30.570Z 27 540-559 118 118 118 177 177 177 F F F
162852023-09-22T23:31:30.573Z 28 560-579 115 115 115 172 172 172 F F F
162862023-09-22T23:31:30.576Z 29 580-599 108 108 108 165 165 165 F F F
162872023-09-22T23:31:30.579ZMax gen: 123, Max flush: 186
162882023-09-22T23:31:30.582Z
162892023-09-22T23:31:30.585ZBegin loop 62
162902023-09-22T23:31:30.588ZDownstairs to restart: 0
162912023-09-22T23:31:30.591Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.525164199Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
162922023-09-22T23:31:30.593Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.525534245Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
162932023-09-22T23:31:30.596Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.525587843Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
162942023-09-22T23:31:30.599Z{"msg":"Crucible stats registered with UUID: f035e4c1-8b5c-4815-a286-6fc18f8a3d9b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.525631735Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
162952023-09-22T23:31:30.602Z{"msg":"Crucible f035e4c1-8b5c-4815-a286-6fc18f8a3d9b has session id: b1c24b83-bcc9-4e31-af9e-23a961c6e015","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.525674879Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
162962023-09-22T23:31:30.604ZCrucible runtime is spawned
162972023-09-22T23:31:30.607ZThe guest has requested activation with gen:124
162982023-09-22T23:31:30.610Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.526057503Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673,"task":"up_listen"}
162992023-09-22T23:31:30.613Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.526120348Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163002023-09-22T23:31:30.615Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.526162798Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163012023-09-22T23:31:30.618Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.526240951Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673,"looper":"2"}
163022023-09-22T23:31:30.621Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.526483993Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673,"looper":"0"}
163032023-09-22T23:31:30.624Z{"msg":"[2] f035e4c1-8b5c-4815-a286-6fc18f8a3d9b looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.526554184Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673,"looper":"2"}
163042023-09-22T23:31:30.627Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.526651317Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673,"looper":"1"}
163052023-09-22T23:31:30.630Z{"msg":"[1] f035e4c1-8b5c-4815-a286-6fc18f8a3d9b looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.526803758Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673,"looper":"1"}
163062023-09-22T23:31:30.634Z{"msg":"f035e4c1-8b5c-4815-a286-6fc18f8a3d9b active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.527044295Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163072023-09-22T23:31:30.637Z{"msg":"Set desired generation to :124","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.52713222Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163082023-09-22T23:31:30.640Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.527187975Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163092023-09-22T23:31:30.643Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.527496041Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163102023-09-22T23:31:30.645Z{"msg":"[2] f035e4c1-8b5c-4815-a286-6fc18f8a3d9b (70b05aea-00cd-4739-b641-adb4b2ef5d83) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.52783456Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163112023-09-22T23:31:30.648Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.527930262Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163122023-09-22T23:31:30.651Z{"msg":"[2] client is_active_req TRUE, promote! session 70b05aea-00cd-4739-b641-adb4b2ef5d83","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.527975228Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163132023-09-22T23:31:30.654Z{"msg":"[1] f035e4c1-8b5c-4815-a286-6fc18f8a3d9b (70b05aea-00cd-4739-b641-adb4b2ef5d83) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.528006339Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163142023-09-22T23:31:30.657Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.528055099Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163152023-09-22T23:31:30.660Z{"msg":"[1] client is_active_req TRUE, promote! session 70b05aea-00cd-4739-b641-adb4b2ef5d83","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.528228451Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163162023-09-22T23:31:30.663Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.528587812Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163172023-09-22T23:31:30.666Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.528655468Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163182023-09-22T23:31:30.668Z{"msg":"f035e4c1-8b5c-4815-a286-6fc18f8a3d9b New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.528700205Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163192023-09-22T23:31:30.671Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.528800031Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163202023-09-22T23:31:30.674Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.528870444Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163212023-09-22T23:31:30.677Z{"msg":"f035e4c1-8b5c-4815-a286-6fc18f8a3d9b New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.528944428Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163222023-09-22T23:31:30.679Z{"msg":"[2] f035e4c1-8b5c-4815-a286-6fc18f8a3d9b (70b05aea-00cd-4739-b641-adb4b2ef5d83) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.530864165Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163232023-09-22T23:31:30.682Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.530934722Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163242023-09-22T23:31:30.685Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:31:30.531053878Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163252023-09-22T23:31:30.688Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.531113727Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163262023-09-22T23:31:30.691Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.531156415Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163272023-09-22T23:31:30.698Z{"msg":"f035e4c1-8b5c-4815-a286-6fc18f8a3d9b New WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.531178284Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163282023-09-22T23:31:30.703Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.531267695Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163292023-09-22T23:31:30.707Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.531314493Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163302023-09-22T23:31:30.710Z{"msg":"[1] f035e4c1-8b5c-4815-a286-6fc18f8a3d9b (70b05aea-00cd-4739-b641-adb4b2ef5d83) New WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.531339986Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163312023-09-22T23:31:30.713Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.531377923Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163322023-09-22T23:31:30.715Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:31:30.531417438Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163332023-09-22T23:31:30.718Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.53145669Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163342023-09-22T23:31:30.721Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.53149844Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163352023-09-22T23:31:30.724Z{"msg":"f035e4c1-8b5c-4815-a286-6fc18f8a3d9b New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.53152046Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163362023-09-22T23:31:30.727Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.53155762Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163372023-09-22T23:31:30.730Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:30.531577083Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163382023-09-22T23:31:31.528Z{"msg":"[0] f035e4c1-8b5c-4815-a286-6fc18f8a3d9b looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.528396412Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673,"looper":"0"}
163392023-09-22T23:31:31.532Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.528487142Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163402023-09-22T23:31:31.535Z{"msg":"[0] f035e4c1-8b5c-4815-a286-6fc18f8a3d9b (70b05aea-00cd-4739-b641-adb4b2ef5d83) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.529706527Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163412023-09-22T23:31:31.538Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.529761765Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163422023-09-22T23:31:31.541Z{"msg":"[0] client is_active_req TRUE, promote! session 70b05aea-00cd-4739-b641-adb4b2ef5d83","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.529805124Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163432023-09-22T23:31:31.544Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.530391364Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163442023-09-22T23:31:31.548Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.530451905Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163452023-09-22T23:31:31.550Z{"msg":"f035e4c1-8b5c-4815-a286-6fc18f8a3d9b WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.530494899Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163462023-09-22T23:31:31.553Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.531659244Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163472023-09-22T23:31:31.557Z{"msg":"[0] f035e4c1-8b5c-4815-a286-6fc18f8a3d9b (70b05aea-00cd-4739-b641-adb4b2ef5d83) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.532883732Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163482023-09-22T23:31:31.559Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.532938605Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163492023-09-22T23:31:31.562Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:31:31.532969072Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163502023-09-22T23:31:31.566Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.533026793Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163512023-09-22T23:31:31.568Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.533070157Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163522023-09-22T23:31:31.572Z{"msg":"f035e4c1-8b5c-4815-a286-6fc18f8a3d9b WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.533110206Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163532023-09-22T23:31:31.575Z{"msg":"[0]R flush_numbers[0..12]: [183, 185, 185, 183, 182, 182, 186, 186, 186, 184, 179, 179]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.533151123Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163542023-09-22T23:31:31.578Z{"msg":"[0]R generation[0..12]: [121, 122, 122, 121, 120, 120, 123, 123, 123, 122, 119, 119]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.533212617Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163552023-09-22T23:31:31.581Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","The guest has finished waiting for activation with:124
163562023-09-22T23:31:31.584Zv":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
163572023-09-22T23:31:31.587Z,"time":"2023-09-22T23:31:31.533267328Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163582023-09-22T23:31:31.590Z{"msg":"[1]R flush_numbers[0..12]: [183, 185, 185, 183, 182, 182, 186, 186, 186, 184, 179, 179]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.533292063Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163592023-09-22T23:31:31.593Z{"msg":"[1]R generation[0..12]: [121, 122, 122, 121, 120, 120, 123, 123, 123, 122, 119, 119]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.533344928Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163602023-09-22T23:31:31.597Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.533413503Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163612023-09-22T23:31:31.599Z{"msg":"[2]R flush_numbers[0..12]: [183, 185, 185, 183, 182, 182, 186, 186, 186, 184, 179, 179]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.533487172Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163622023-09-22T23:31:31.602Z{"msg":"[2]R generation[0..12]: [121, 122, 122, 121, 120, 120, 123, 123, 123, 122, 119, 119]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.533557165Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163632023-09-22T23:31:31.605Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.533627874Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163642023-09-22T23:31:31.608Z{"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.53368949Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163652023-09-22T23:31:31.611Z{"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.533745963Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163662023-09-22T23:31:31.614Z{"msg":"Next flush: 187","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.533787648Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163672023-09-22T23:31:31.617Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.533835598Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163682023-09-22T23:31:31.622Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.533871329Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163692023-09-22T23:31:31.625Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.533923136Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163702023-09-22T23:31:31.628Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.533964633Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163712023-09-22T23:31:31.630Z{"msg":"f035e4c1-8b5c-4815-a286-6fc18f8a3d9b is now active with session: 70b05aea-00cd-4739-b641-adb4b2ef5d83","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.533995508Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163722023-09-22T23:31:31.633Z{"msg":"f035e4c1-8b5c-4815-a286-6fc18f8a3d9b Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.5340598Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163732023-09-22T23:31:31.636Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.534120541Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163742023-09-22T23:31:31.639Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
163752023-09-22T23:31:31.642Z,"time":"2023-09-22T23:31:31.534186562Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163762023-09-22T23:31:31.645Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.534301571Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163772023-09-22T23:31:31.648Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.534347243Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163782023-09-22T23:31:31.651Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.534385672Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163792023-09-22T23:31:31.654Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.534423303Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163802023-09-22T23:31:31.657Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.534461267Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163812023-09-22T23:31:31.660Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.534499204Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163822023-09-22T23:31:31.663Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.53453689Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163832023-09-22T23:31:31.666Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.534575592Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163842023-09-22T23:31:31.669Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.534613491Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1673}
163852023-09-22T23:31:31.672ZLoading write count information from file "/tmp/test_repair_verify.data"
163862023-09-22T23:31:31.675ZRun Repair workload
163872023-09-22T23:31:31.678Z01/30 Write block 517 len 8192 data: 13 14
163882023-09-22T23:31:31.680Z02/30 Read block 206 len 28672
163892023-09-22T23:31:31.684Z03/30 Flush
163902023-09-22T23:31:31.686Z04/30 Write block 121 len 28672 data: 10 12 12 10 8 6 10
163912023-09-22T23:31:31.691Z05/30 Read block 261 len 16384
163922023-09-22T23:31:31.693Z06/30 Write block 337 len 24576 data: 7 8 7 6 5 9
163932023-09-22T23:31:31.696Z07/30 Flush
163942023-09-22T23:31:31.699Z08/30 Write block 438 len 32768 data: 10 8 8 11 11 13 14 13
163952023-09-22T23:31:31.702Z09/30 Flush
163962023-09-22T23:31:31.705Z10/30 Write block 205 len 32768 data: 17 15 14 16 13 14 16 12
163972023-09-22T23:31:31.708Z11/30 Write block 581 len 36864 data: 8 8 7 7 7 5 5 7 8
163982023-09-22T23:31:31.711Z12/30 Read block 536 len 32768
163992023-09-22T23:31:31.715Z13/30 Read block 173 len 24576
164002023-09-22T23:31:31.728Z14/30 Write block 104 len 20480 data: 15 17 15 13 14
164012023-09-22T23:31:31.736Z15/30 Write block 263 len 40960 data: 13 13 12 12 15 12 13 13 16 17
164022023-09-22T23:31:31.743Z16/30 Read block 236 len 28672
164032023-09-22T23:31:31.746Z17/30 Write block 100 len 36864 data: 11 11 12 13 16 18 16 14 15
164042023-09-22T23:31:31.750Z18/30 Read block 170 len 36864
164052023-09-22T23:31:31.753Z19/30 Write block 519 len 20480 data: 12 12 10 10 10
164062023-09-22T23:31:31.757Z20/30 Read block 345 len 16384
164072023-09-22T23:31:31.760Z21/30 Read block 514 len 40960
164082023-09-22T23:31:31.765Z22/30 Read block 193 len 24576
164092023-09-22T23:31:31.768Z23/30 Read block 468 len 40960
164102023-09-22T23:31:31.771Z24/30 Read block 474 len 28672
164112023-09-22T23:31:31.774Z25/30 Read block 530 len 20480
164122023-09-22T23:31:31.777Z26/30 Read block 101 len 8192
164132023-09-22T23:31:31.780Z27/30 Read block 63 len 28672
164142023-09-22T23:31:31.783Z28/30 Read block 331 len 32768
164152023-09-22T23:31:31.786Z29/30 Read block 380 len 8192
164162023-09-22T23:31:31.789Z30/30 Write block 35 len 16384 data: 11 14 14 13
164172023-09-22T23:31:31.792Z----------------------------------------------------------------
164182023-09-22T23:31:31.795Z Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:27
164192023-09-22T23:31:31.797ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
164202023-09-22T23:31:31.800Z 4 Acked 1003 Write 7 Sent Done Done false
164212023-09-22T23:31:31.803Z 5 Acked 1004 Read 4 Sent Done Done false
164222023-09-22T23:31:31.806Z 6 Acked 1005 Write 6 Sent Done Done false
164232023-09-22T23:31:31.809Z 7 Acked 1006 Flush 0 Sent Done Done false
164242023-09-22T23:31:31.812Z 8 Acked 1007 Write 8 Sent Done Done false
164252023-09-22T23:31:31.815Z 9 Acked 1008 Flush 0 Sent Done Done false
164262023-09-22T23:31:31.818Z 10 Acked 1009 Write 8 Sent Done Done false
164272023-09-22T23:31:31.820Z 11 Acked 1010 Write 9 Sent Done Done false
164282023-09-22T23:31:31.823Z 12 Acked 1011 Read 8 Sent Done Done false
164292023-09-22T23:31:31.826Z 13 Acked 1012 Read 6 Sent Done Done false
164302023-09-22T23:31:31.829Z 14 Acked 1013 Write 5 Sent Done Done false
164312023-09-22T23:31:31.832Z 15 Acked 1014 Write 10 Sent Done Done false
164322023-09-22T23:31:31.834Z 16 Acked 1015 Read 7 Sent Done Done false
164332023-09-22T23:31:31.837Z 17 Acked 1016 Write 9 Sent Done Done false
164342023-09-22T23:31:31.840Z 18 Acked 1017 Read 9 Sent Done Done false
164352023-09-22T23:31:31.843Z 19 Acked 1018 Write 5 Sent Done Done false
164362023-09-22T23:31:31.846Z 20 Acked 1019 Read 4 Sent Done Done false
164372023-09-22T23:31:31.849Z 21 Acked 1020 Read 10 Sent Done Done false
164382023-09-22T23:31:31.852Z 22 Acked 1021 Read 6 Sent Done Done false
164392023-09-22T23:31:31.855Z 23 Acked 1022 Read 10 Sent Done Done false
164402023-09-22T23:31:31.858Z 24 Acked 1023 Read 7 Sent Done Done false
164412023-09-22T23:31:31.862Z 25 Acked 1024 Read 5 Sent Done Done false
164422023-09-22T23:31:31.865Z 26 Acked 1025 Read 2 Sent Done Done false
164432023-09-22T23:31:31.870Z 27 Acked 1026 Read 7 Sent Done Done false
164442023-09-22T23:31:31.874Z 28 Acked 1027 Read 8 Sent Done Done false
164452023-09-22T23:31:31.876Z 29 Acked 1028 Read 2 Sent Done Done false
164462023-09-22T23:31:31.879Z 30 Acked 1029 Write 4 Sent Sent Sent false
164472023-09-22T23:31:31.882Z STATES DS:0 DS:1 DS:2 TOTAL
164482023-09-22T23:31:31.885Z New 0 0 0 0
164492023-09-22T23:31:31.888Z Sent 27 1 1 29
164502023-09-22T23:31:31.891Z Done 0 26 26 52
164512023-09-22T23:31:31.894Z Skipped 0 0 0 0
164522023-09-22T23:31:31.897Z Error 0 0 0 0
164532023-09-22T23:31:31.900ZLast Flush: 1002 1008 1008
164542023-09-22T23:31:31.903ZDownstairs last five completed: 1002 1001 1000
164552023-09-22T23:31:31.908ZUpstairs last five completed: 30 29 28 27 26 25
164562023-09-22T23:31:31.911ZWrote out file "/tmp/test_repair_verify.data"
164572023-09-22T23:31:31.914Z
164582023-09-22T23:31:31.916ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
164592023-09-22T23:31:31.919Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.678757874Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1674}
164602023-09-22T23:31:31.922Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.679084371Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1674}
164612023-09-22T23:31:31.925Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.679108591Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1674}
164622023-09-22T23:31:31.928Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.704464074Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1674}
164632023-09-22T23:31:31.931Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.70449684Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1674}
164642023-09-22T23:31:31.936Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.704516567Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1674}
164652023-09-22T23:31:31.939Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.72597834Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1674}
164662023-09-22T23:31:31.942Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.726032276Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1674}
164672023-09-22T23:31:31.945Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:31.726053636Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1674}
164682023-09-22T23:31:31.948ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
164692023-09-22T23:31:31.950ZError: Difference in extent metadata found!
164702023-09-22T23:31:31.953Z 0 000-019 121 121 121 183 183 183 F F F
164712023-09-22T23:31:31.956Z 1 020-039 122 122 122 185 185 185 F T T <---
164722023-09-22T23:31:31.959Z 2 040-059 122 122 122 185 185 185 F F F
164732023-09-22T23:31:31.962Z 3 060-079 121 121 121 183 183 183 F F F
164742023-09-22T23:31:31.965Z 4 080-099 120 120 120 182 182 182 F F F
164752023-09-22T23:31:31.968Z 5 100-119 120 120 120 182 182 182 F T T <---
164762023-09-22T23:31:31.970Z 6 120-139 123 124 124 186 188 188 F F F <---
164772023-09-22T23:31:31.973Z 7 140-159 123 123 123 186 186 186 F F F
164782023-09-22T23:31:31.976Z 8 160-179 123 123 123 186 186 186 F F F
164792023-09-22T23:31:31.979Z 9 180-199 122 122 122 184 184 184 F F F
164802023-09-22T23:31:31.981Z 10 200-219 119 119 119 179 179 179 F T T <---
164812023-09-22T23:31:31.984Z 11 220-239 119 119 119 179 179 179 F F F
164822023-09-22T23:31:31.987Z 12 240-259 114 114 114 171 171 171 F F F
164832023-09-22T23:31:31.990Z 13 260-279 118 118 118 177 177 177 F T T <---
164842023-09-22T23:31:31.993Z 14 280-299 122 122 122 184 184 184 F F F
164852023-09-22T23:31:31.995Z 15 300-319 121 121 121 183 183 183 F F F
164862023-09-22T23:31:31.998Z 16 320-339 121 124 124 183 188 188 F F F <---
164872023-09-22T23:31:32.001Z 17 340-359 123 124 124 186 188 188 F F F <---
164882023-09-22T23:31:32.004Z 18 360-379 123 123 123 186 186 186 F F F
164892023-09-22T23:31:32.007Z 19 380-399 121 121 121 183 183 183 F F F
164902023-09-22T23:31:32.011Z 20 400-419 118 118 118 176 176 176 F F F
164912023-09-22T23:31:32.019Z 21 420-439 114 124 124 171 189 189 F F F <---
164922023-09-22T23:31:32.433Z 22 440-459 123 124 124 186 189 189 F F F <---
164932023-09-22T23:31:32.436Z 23 460-479 123 123 123 186 186 186 F F F
164942023-09-22T23:31:32.439Z 24 480-499 121 121 121 183 183 183 F F F
164952023-09-22T23:31:32.442Z 25 500-519 124 124 124 187 187 187 F T T <---
164962023-09-22T23:31:32.451Z 26 520-539 123 123 123 186 186 186 F T T <---
164972023-09-22T23:31:32.463Z 27 540-559 118 118 118 177 177 177 F F F
164982023-09-22T23:31:32.485Z 28 560-579 115 115 115 172 172 172 F F F
164992023-09-22T23:31:32.509Z 29 580-599 108 108 108 165 165 165 F T T <---
165002023-09-22T23:31:32.523ZMax gen: 124, Max flush: 189
165012023-09-22T23:31:32.534ZA Difference in extent metadata is expected here
165022023-09-22T23:31:32.541ZOn loop 62
165032023-09-22T23:31:32.556Z
165042023-09-22T23:31:32.571ZVerifying data now
165052023-09-22T23:31:38.087ZNo change after verify
165062023-09-22T23:31:38.090ZLoop: 62 Downstairs dump after verify (and repair):
165072023-09-22T23:31:38.105Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.105775637Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1679}
165082023-09-22T23:31:38.109Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.106093709Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1679}
165092023-09-22T23:31:38.113Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.106119068Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1679}
165102023-09-22T23:31:38.126Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.124618949Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1679}
165112023-09-22T23:31:38.537Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.124672273Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1679}
165122023-09-22T23:31:38.541Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.124693426Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1679}
165132023-09-22T23:31:38.545Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.13547186Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1679}
165142023-09-22T23:31:38.551Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.135526211Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1679}
165152023-09-22T23:31:38.558Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.135565797Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1679}
165162023-09-22T23:31:38.563ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
165172023-09-22T23:31:38.569Z 0 000-019 121 121 121 183 183 183 F F F
165182023-09-22T23:31:38.574Z 1 020-039 125 125 125 190 190 190 F F F
165192023-09-22T23:31:38.582Z 2 040-059 122 122 122 185 185 185 F F F
165202023-09-22T23:31:38.589Z 3 060-079 121 121 121 183 183 183 F F F
165212023-09-22T23:31:38.596Z 4 080-099 120 120 120 182 182 182 F F F
165222023-09-22T23:31:38.602Z 5 100-119 125 125 125 190 190 190 F F F
165232023-09-22T23:31:38.608Z 6 120-139 124 124 124 188 188 188 F F F
165242023-09-22T23:31:38.617Z 7 140-159 123 123 123 186 186 186 F F F
165252023-09-22T23:31:38.625Z 8 160-179 123 123 123 186 186 186 F F F
165262023-09-22T23:31:38.630Z 9 180-199 122 122 122 184 184 184 F F F
165272023-09-22T23:31:38.633Z 10 200-219 125 125 125 190 190 190 F F F
165282023-09-22T23:31:38.636Z 11 220-239 119 119 119 179 179 179 F F F
165292023-09-22T23:31:38.639Z 12 240-259 114 114 114 171 171 171 F F F
165302023-09-22T23:31:38.642Z 13 260-279 125 125 125 190 190 190 F F F
165312023-09-22T23:31:38.644Z 14 280-299 122 122 122 184 184 184 F F F
165322023-09-22T23:31:38.647Z 15 300-319 121 121 121 183 183 183 F F F
165332023-09-22T23:31:38.650Z 16 320-339 124 124 124 188 188 188 F F F
165342023-09-22T23:31:38.653Z 17 340-359 124 124 124 188 188 188 F F F
165352023-09-22T23:31:38.656Z 18 360-379 123 123 123 186 186 186 F F F
165362023-09-22T23:31:38.660Z 19 380-399 121 121 121 183 183 183 F F F
165372023-09-22T23:31:38.664Z 20 400-419 118 118 118 176 176 176 F F F
165382023-09-22T23:31:38.677Z 21 420-439 124 124 124 189 189 189 F F F
165392023-09-22T23:31:39.099Z 22 440-459 124 124 124 189 189 189 F F F
165402023-09-22T23:31:39.102Z 23 460-479 123 123 123 186 186 186 F F F
165412023-09-22T23:31:39.106Z 24 480-499 121 121 121 183 183 183 F F F
165422023-09-22T23:31:39.110Z 25 500-519 125 125 125 190 190 190 F F F
165432023-09-22T23:31:39.119Z 26 520-539 125 125 125 190 190 190 F F F
165442023-09-22T23:31:39.548Z 27 540-559 118 118 118 177 177 177 F F F
165452023-09-22T23:31:39.554Z 28 560-579 115 115 115 172 172 172 F F F
165462023-09-22T23:31:39.561Z 29 580-599 125 125 125 190 190 190 F F F
165472023-09-22T23:31:39.567ZMax gen: 125, Max flush: 190
165482023-09-22T23:31:39.576Z
165492023-09-22T23:31:39.596ZBegin loop 63
165502023-09-22T23:31:39.610ZDownstairs to restart: 0
165512023-09-22T23:31:39.626Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.184395122Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165522023-09-22T23:31:39.636Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.184767845Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165532023-09-22T23:31:39.643Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.184822519Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165542023-09-22T23:31:39.648Z{"msg":"Crucible stats registered with UUID: 16f9ba44-9e0f-4a69-ba69-c96175ee0406","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.18486618Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165552023-09-22T23:31:39.653Z{"msg":"Crucible 16f9ba44-9e0f-4a69-ba69-c96175ee0406 has session id: 23b7b869-dc1b-4f0f-8454-f1120d8734c2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.184939465Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165562023-09-22T23:31:39.658ZCrucible runtime is spawned
165572023-09-22T23:31:39.662Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30The guest has requested activation with gen:126
165582023-09-22T23:31:39.667Z,"time":"2023-09-22T23:31:38.185130188Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681,"task":"up_listen"}
165592023-09-22T23:31:39.671Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.185283148Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165602023-09-22T23:31:39.676Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.185330395Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165612023-09-22T23:31:39.679Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.185370732Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681,"looper":"0"}
165622023-09-22T23:31:39.682Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.185639494Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681,"looper":"1"}
165632023-09-22T23:31:39.684Z{"msg":"[1] 16f9ba44-9e0f-4a69-ba69-c96175ee0406 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.185893882Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681,"looper":"1"}
165642023-09-22T23:31:39.688Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.185960662Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681,"looper":"2"}
165652023-09-22T23:31:39.691Z{"msg":"[2] 16f9ba44-9e0f-4a69-ba69-c96175ee0406 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.186172812Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681,"looper":"2"}
165662023-09-22T23:31:39.693Z{"msg":"16f9ba44-9e0f-4a69-ba69-c96175ee0406 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.186555954Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165672023-09-22T23:31:39.697Z{"msg":"Set desired generation to :126","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.186610191Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165682023-09-22T23:31:39.699Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.18675209Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165692023-09-22T23:31:39.702Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.1872188Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165702023-09-22T23:31:39.706Z{"msg":"[1] 16f9ba44-9e0f-4a69-ba69-c96175ee0406 (40ad860e-9db7-4c45-b1e5-a9e0063d4848) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.187545968Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165712023-09-22T23:31:39.710Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.187614057Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165722023-09-22T23:31:39.717Z{"msg":"[1] client is_active_req TRUE, promote! session 40ad860e-9db7-4c45-b1e5-a9e0063d4848","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.187697902Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165732023-09-22T23:31:39.722Z{"msg":"[2] 16f9ba44-9e0f-4a69-ba69-c96175ee0406 (40ad860e-9db7-4c45-b1e5-a9e0063d4848) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.187777008Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165742023-09-22T23:31:39.728Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.187907054Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165752023-09-22T23:31:39.733Z{"msg":"[2] client is_active_req TRUE, promote! session 40ad860e-9db7-4c45-b1e5-a9e0063d4848","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.187983688Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165762023-09-22T23:31:39.739Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.188229749Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165772023-09-22T23:31:39.744Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.1883069Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165782023-09-22T23:31:39.749Z{"msg":"16f9ba44-9e0f-4a69-ba69-c96175ee0406 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.188336569Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165792023-09-22T23:31:39.756Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.188445665Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165802023-09-22T23:31:39.760Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.188477115Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165812023-09-22T23:31:39.765Z{"msg":"16f9ba44-9e0f-4a69-ba69-c96175ee0406 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.188496955Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165822023-09-22T23:31:39.769Z{"msg":"[1] 16f9ba44-9e0f-4a69-ba69-c96175ee0406 (40ad860e-9db7-4c45-b1e5-a9e0063d4848) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.191079788Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165832023-09-22T23:31:39.774Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.191114227Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165842023-09-22T23:31:39.780Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:31:38.191132823Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165852023-09-22T23:31:39.784Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.19115019Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165862023-09-22T23:31:39.790Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.191187904Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165872023-09-22T23:31:39.794Z{"msg":"16f9ba44-9e0f-4a69-ba69-c96175ee0406 New WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.191274414Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165882023-09-22T23:31:39.800Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.191299248Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165892023-09-22T23:31:39.805Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.191316735Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165902023-09-22T23:31:39.812Z{"msg":"[2] 16f9ba44-9e0f-4a69-ba69-c96175ee0406 (40ad860e-9db7-4c45-b1e5-a9e0063d4848) New WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.19141554Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165912023-09-22T23:31:39.817Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.191441495Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165922023-09-22T23:31:39.822Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:31:38.19145982Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165932023-09-22T23:31:39.828Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.19147755Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165942023-09-22T23:31:39.839Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.191495889Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165952023-09-22T23:31:39.849Z{"msg":"16f9ba44-9e0f-4a69-ba69-c96175ee0406 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.191515005Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165962023-09-22T23:31:39.854Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.191532559Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165972023-09-22T23:31:39.861Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:38.191549774Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
165982023-09-22T23:31:39.868Z{"msg":"[0] 16f9ba44-9e0f-4a69-ba69-c96175ee0406 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.186703516Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681,"looper":"0"}
165992023-09-22T23:31:39.874Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.186789663Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166002023-09-22T23:31:39.880Z{"msg":"[0] 16f9ba44-9e0f-4a69-ba69-c96175ee0406 (40ad860e-9db7-4c45-b1e5-a9e0063d4848) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.187891586Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166012023-09-22T23:31:39.885Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.187943573Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166022023-09-22T23:31:39.889Z{"msg":"[0] client is_active_req TRUE, promote! session 40ad860e-9db7-4c45-b1e5-a9e0063d4848","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.187986817Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166032023-09-22T23:31:39.895Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.18857476Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166042023-09-22T23:31:39.904Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.188631671Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166052023-09-22T23:31:39.912Z{"msg":"16f9ba44-9e0f-4a69-ba69-c96175ee0406 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.188675704Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166062023-09-22T23:31:39.920Z{"msg":"[0] 16f9ba44-9e0f-4a69-ba69-c96175ee0406 (40ad860e-9db7-4c45-b1e5-a9e0063d4848) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.191055834Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166072023-09-22T23:31:39.924Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.191120836Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166082023-09-22T23:31:39.932Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:31:39.191142705Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166092023-09-22T23:31:39.939Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.191160429Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166102023-09-22T23:31:39.946ZError: Difference in extent metadata found!
166112023-09-22T23:31:39.952Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.191198301Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166122023-09-22T23:31:39.959Z{"msg":"16f9ba44-9e0f-4a69-ba69-c96175ee0406 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.19121978Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166132023-09-22T23:31:39.964Z{"msg":"[0]R flush_numbers[0..12]: [183, 190, 185, 183, 182, 190, 188, 186, 186, 184, 190, 179]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.191239676Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166142023-09-22T23:31:39.968Z{"msg":"[0]R generation[0..12]: [121, 125, 122, 121, 120, 125, 124, 123, 123, 122, 125, 119]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.191259045Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166152023-09-22T23:31:39.977Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.191278926Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166162023-09-22T23:31:39.982Z{"msg":"[1]R flush_numbers[0..12]: [183, 190, 185, 183, 182, 190, 188, 186, 186, 184, 190, 179]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.191298171Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166172023-09-22T23:31:40.390Z{"msg":"[1]R generation[0..12]: [121, 125, 122, 121, 120, 125, 124, 123, 123, 122, 125, 119]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.191317542Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166182023-09-22T23:31:40.394Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.191337434Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166192023-09-22T23:31:40.397Z{"msg":"[2]R flush_numbers[0..12]: [183, 190, 185, 183, 182, 190, 188, 186, 186, 184, 190, 179]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.191356748Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166202023-09-22T23:31:40.406Z{"msg":"[2]R generation[0..12]: [121, 125, 122, 121, 120, 125, 124, 123, 123, 122, 125, 119]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.191375801Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166212023-09-22T23:31:40.411Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.191395804Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166222023-09-22T23:31:40.414Z{"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.191413239Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166232023-09-22T23:31:40.417Z{"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.191431196Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166242023-09-22T23:31:40.421Z{"msg":"Next flush: 191","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.191448266Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166252023-09-22T23:31:40.424Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.191465583Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166262023-09-22T23:31:40.427Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:126
166272023-09-22T23:31:40.430Z,"time":"2023-09-22T23:31:39.191482848Z","hostname":"Wait for a query_work_queue command to finish before sending IO
166282023-09-22T23:31:40.434Zip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166292023-09-22T23:31:40.439Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.191503905Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166302023-09-22T23:31:40.443Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.1915218Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166312023-09-22T23:31:40.451Z{"msg":"16f9ba44-9e0f-4a69-ba69-c96175ee0406 is now active with session: 40ad860e-9db7-4c45-b1e5-a9e0063d4848","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.191541939Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166322023-09-22T23:31:40.462Z{"msg":"16f9ba44-9e0f-4a69-ba69-c96175ee0406 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.19156036Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166332023-09-22T23:31:40.475Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.191578427Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166342023-09-22T23:31:40.512Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.191595516Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166352023-09-22T23:31:40.520Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.191632462Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166362023-09-22T23:31:40.523Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.19166346Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166372023-09-22T23:31:40.526Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.191698129Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166382023-09-22T23:31:40.529Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.191925126Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166392023-09-22T23:31:40.532Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.191967995Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166402023-09-22T23:31:40.535Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.192003181Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166412023-09-22T23:31:40.538Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.192039458Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166422023-09-22T23:31:40.541Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.192074167Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166432023-09-22T23:31:40.544Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.1921274Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1681}
166442023-09-22T23:31:40.548ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
166452023-09-22T23:31:40.551ZLoading write count information from file "/tmp/test_repair_verify.data"
166462023-09-22T23:31:40.554ZRun Repair workload
166472023-09-22T23:31:40.557Z01/30 Write block 448 len 16384 data: 10 11 12 11
166482023-09-22T23:31:40.560Z02/30 Write block 143 len 4096 data: 9
166492023-09-22T23:31:40.563Z03/30 Write block 244 len 24576 data: 6 7 4 5 5 6
166502023-09-22T23:31:40.566Z04/30 Write block 562 len 24576 data: 9 10 12 13 13 14
166512023-09-22T23:31:40.568Z05/30 Read block 286 len 12288
166522023-09-22T23:31:40.571Z06/30 Read block 431 len 28672
166532023-09-22T23:31:40.574Z07/30 Read block 47 len 32768
166542023-09-22T23:31:40.577Z08/30 Read block 166 len 16384
166552023-09-22T23:31:40.579Z09/30 Write block 567 len 12288 data: 15 15 15
166562023-09-22T23:31:40.582Z10/30 Read block 243 len 40960
166572023-09-22T23:31:40.586Z11/30 Write block 107 len 36864 data: 15 16 15 15 12 11 11 9 8
166582023-09-22T23:31:40.589Z12/30 Write block 578 len 4096 data: 9
166592023-09-22T23:31:40.592Z13/30 Read block 132 len 16384
166602023-09-22T23:31:40.595Z14/30 Read block 437 len 12288
166612023-09-22T23:31:40.598Z15/30 Read block 41 len 8192
166622023-09-22T23:31:40.601Z16/30 Write block 160 len 28672 data: 12 12 12 10 9 8 6
166632023-09-22T23:31:40.605Z17/30 Read block 252 len 16384
166642023-09-22T23:31:40.608Z18/30 Write block 165 len 24576 data: 9 7 7 6 6 10
166652023-09-22T23:31:40.611Z19/30 Write block 100 len 36864 data: 12 12 13 14 17 19 17 16 17
166662023-09-22T23:31:40.614Z20/30 Read block 229 len 36864
166672023-09-22T23:31:40.617Z21/30 Write block 463 len 24576 data: 10 11 15 12 11 12
166682023-09-22T23:31:40.621Z22/30 Read block 19 len 4096
166692023-09-22T23:31:40.626Z23/30 Write block 335 len 28672 data: 12 11 8 9 8 7 6
166702023-09-22T23:31:40.631Z24/30 Read block 428 len 32768
166712023-09-22T23:31:40.634Z25/30 Read block 21 len 12288
166722023-09-22T23:31:40.637Z26/30 Flush
166732023-09-22T23:31:40.640Z27/30 Write block 568 len 4096 data: 16
166742023-09-22T23:31:40.643Z28/30 Write block 64 len 32768 data: 16 19 20 19 19 17 10 11
166752023-09-22T23:31:40.646Z29/30 Write block 68 len 20480 data: 20 18 11 12 11
166762023-09-22T23:31:40.651Z30/30 Read block 289 len 32768
166772023-09-22T23:31:40.654Z----------------------------------------------------------------
166782023-09-22T23:31:40.657Z Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30
166792023-09-22T23:31:40.660ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
166802023-09-22T23:31:40.663Z 1 Acked 1000 Write 4 Sent Done Done false
166812023-09-22T23:31:40.666Z 2 Acked 1001 Write 1 Sent Done Done false
166822023-09-22T23:31:40.669Z 3 Acked 1002 Write 6 Sent Done Done false
166832023-09-22T23:31:40.672Z 4 Acked 1003 Write 6 Sent Done Done false
166842023-09-22T23:31:40.676Z 5 Acked 1004 Read 3 Sent Done Done false
166852023-09-22T23:31:40.679Z 6 Acked 1005 Read 7 Sent Done Done false
166862023-09-22T23:31:40.682Z 7 Acked 1006 Read 8 Sent Done Done false
166872023-09-22T23:31:40.686Z 8 Acked 1007 Read 4 Sent Done Done false
166882023-09-22T23:31:40.688Z 9 Acked 1008 Write 3 Sent Done Done false
166892023-09-22T23:31:40.691Z 10 Acked 1009 Read 10 Sent Done Done false
166902023-09-22T23:31:40.694Z 11 Acked 1010 Write 9 Sent Done Done false
166912023-09-22T23:31:40.697Z 12 Acked 1011 Write 1 Sent Done Done false
166922023-09-22T23:31:40.700Z 13 Acked 1012 Read 4 Sent Done Done false
166932023-09-22T23:31:40.703Z 14 Acked 1013 Read 3 Sent Done Done false
166942023-09-22T23:31:40.706Z 15 Acked 1014 Read 2 Sent Done Done false
166952023-09-22T23:31:40.709Z 16 Acked 1015 Write 7 Sent Done Done false
166962023-09-22T23:31:40.712Z 17 Acked 1016 Read 4 Sent Done Done false
166972023-09-22T23:31:40.714Z 18 Acked 1017 Write 6 Sent Done Done false
166982023-09-22T23:31:40.717Z 19 Acked 1018 Write 9 Sent Done Done false
166992023-09-22T23:31:40.720Z 20 Acked 1019 Read 9 Sent Done Done false
167002023-09-22T23:31:40.724Z 21 Acked 1020 Write 6 Sent Done Done false
167012023-09-22T23:31:40.727Z 22 Acked 1021 Read 1 Sent Done Done false
167022023-09-22T23:31:40.730Z 23 Acked 1022 Write 7 Sent Done Done false
167032023-09-22T23:31:40.733Z 24 Acked 1023 Read 8 Sent Done Done false
167042023-09-22T23:31:40.736Z 25 Acked 1024 Read 3 Sent Done Done false
167052023-09-22T23:31:40.739Z 26 Acked 1025 Flush 0 Sent Done Done false
167062023-09-22T23:31:40.741Z 27 Acked 1026 Write 1 Sent Done Done false
167072023-09-22T23:31:40.744Z 28 Acked 1027 Write 8 Sent Done Done false
167082023-09-22T23:31:40.747Z 29 Acked 1028 Write 5 Sent Done Done false
167092023-09-22T23:31:40.750Z 30 Acked 1029 Read 8 Sent Done Done false
167102023-09-22T23:31:40.753Z STATES DS:0 DS:1 DS:2 TOTAL
167112023-09-22T23:31:40.756Z New 0 0 0 0
167122023-09-22T23:31:40.759Z Sent 30 0 0 30
167132023-09-22T23:31:40.762Z Done 0 30 30 60
167142023-09-22T23:31:40.765Z Skipped 0 0 0 0
167152023-09-22T23:31:40.768Z Error 0 0 0 0
167162023-09-22T23:31:40.771ZLast Flush: 0 1025 1025
167172023-09-22T23:31:40.775ZDownstairs last five completed:
167182023-09-22T23:31:40.778ZUpstairs last five completed: 30 29 28 27 26 25
167192023-09-22T23:31:40.780ZWrote out file "/tmp/test_repair_verify.data"
167202023-09-22T23:31:40.783Z
167212023-09-22T23:31:40.786ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
167222023-09-22T23:31:40.789Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.325903721Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1682}
167232023-09-22T23:31:40.792Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.326296674Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1682}
167242023-09-22T23:31:40.795Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.326320162Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1682}
167252023-09-22T23:31:40.798Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.348590507Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1682}
167262023-09-22T23:31:40.801Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.348617268Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1682}
167272023-09-22T23:31:40.804Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.348636097Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1682}
167282023-09-22T23:31:40.807Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.364595886Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1682}
167292023-09-22T23:31:40.810Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.364630693Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1682}
167302023-09-22T23:31:40.813Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:39.364650076Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1682}
167312023-09-22T23:31:40.816ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
167322023-09-22T23:31:40.819Z 0 000-019 121 121 121 183 183 183 F F F
167332023-09-22T23:31:40.822Z 1 020-039 125 125 125 190 190 190 F F F
167342023-09-22T23:31:40.825Z 2 040-059 122 122 122 185 185 185 F F F
167352023-09-22T23:31:40.828Z 3 060-079 121 121 121 183 183 183 F T T <---
167362023-09-22T23:31:40.831Z 4 080-099 120 120 120 182 182 182 F F F
167372023-09-22T23:31:40.834Z 5 100-119 125 126 126 190 191 191 F F F <---
167382023-09-22T23:31:40.836Z 6 120-139 124 124 124 188 188 188 F F F
167392023-09-22T23:31:40.839Z 7 140-159 123 126 126 186 191 191 F F F <---
167402023-09-22T23:31:40.842Z 8 160-179 123 126 126 186 191 191 F F F <---
167412023-09-22T23:31:40.845Z 9 180-199 122 122 122 184 184 184 F F F
167422023-09-22T23:31:40.848Z 10 200-219 125 125 125 190 190 190 F F F
167432023-09-22T23:31:40.851Z 11 220-239 119 119 119 179 179 179 F F F
167442023-09-22T23:31:40.855Z 12 240-259 114 126 126 171 191 191 F F F <---
167452023-09-22T23:31:40.858Z 13 260-279 125 125 125 190 190 190 F F F
167462023-09-22T23:31:40.861Z 14 280-299 122 122 122 184 184 184 F F F
167472023-09-22T23:31:40.864Z 15 300-319 121 121 121 183 183 183 F F F
167482023-09-22T23:31:40.867Z 16 320-339 124 126 126 188 191 191 F F F <---
167492023-09-22T23:31:40.870Z 17 340-359 124 126 126 188 191 191 F F F <---
167502023-09-22T23:31:40.873Z 18 360-379 123 123 123 186 186 186 F F F
167512023-09-22T23:31:40.876Z 19 380-399 121 121 121 183 183 183 F F F
167522023-09-22T23:31:40.879Z 20 400-419 118 118 118 176 176 176 F F F
167532023-09-22T23:31:40.883Z 21 420-439 124 124 124 189 189 189 F F F
167542023-09-22T23:31:40.886Z 22 440-459 124 126 126 189 191 191 F F F <---
167552023-09-22T23:31:40.890Z 23 460-479 123 126 126 186 191 191 F F F <---
167562023-09-22T23:31:40.893Z 24 480-499 121 121 121 183 183 183 F F F
167572023-09-22T23:31:40.897Z 25 500-519 125 125 125 190 190 190 F F F
167582023-09-22T23:31:40.900Z 26 520-539 125 125 125 190 190 190 F F F
167592023-09-22T23:31:40.903Z 27 540-559 118 118 118 177 177 177 F F F
167602023-09-22T23:31:40.906Z 28 560-579 115 126 126 172 191 191 F T T <---
167612023-09-22T23:31:40.911Z 29 580-599 125 125 125 190 190 190 F F F
167622023-09-22T23:31:40.914ZMax gen: 126, Max flush: 191
167632023-09-22T23:31:40.916ZA Difference in extent metadata is expected here
167642023-09-22T23:31:40.919ZOn loop 63
167652023-09-22T23:31:40.923Z
167662023-09-22T23:31:40.932ZVerifying data now
167672023-09-22T23:31:45.538ZNo change after verify
167682023-09-22T23:31:45.541ZLoop: 63 Downstairs dump after verify (and repair):
167692023-09-22T23:31:45.556Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.556608456Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1687}
167702023-09-22T23:31:45.559Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.556933179Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1687}
167712023-09-22T23:31:45.562Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.55695819Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1687}
167722023-09-22T23:31:45.575Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.575549366Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1687}
167732023-09-22T23:31:45.577Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.575603877Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1687}
167742023-09-22T23:31:45.585Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.575624495Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1687}
167752023-09-22T23:31:45.588Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.586419173Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1687}
167762023-09-22T23:31:45.591Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.586472022Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1687}
167772023-09-22T23:31:45.594Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.586510367Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1687}
167782023-09-22T23:31:45.597ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
167792023-09-22T23:31:45.600Z 0 000-019 121 121 121 183 183 183 F F F
167802023-09-22T23:31:45.603Z 1 020-039 125 125 125 190 190 190 F F F
167812023-09-22T23:31:45.606Z 2 040-059 122 122 122 185 185 185 F F F
167822023-09-22T23:31:45.609Z 3 060-079 127 127 127 192 192 192 F F F
167832023-09-22T23:31:45.612Z 4 080-099 120 120 120 182 182 182 F F F
167842023-09-22T23:31:45.615Z 5 100-119 126 126 126 191 191 191 F F F
167852023-09-22T23:31:45.618Z 6 120-139 124 124 124 188 188 188 F F F
167862023-09-22T23:31:45.621Z 7 140-159 126 126 126 191 191 191 F F F
167872023-09-22T23:31:45.624Z 8 160-179 126 126 126 191 191 191 F F F
167882023-09-22T23:31:45.626Z 9 180-199 122 122 122 184 184 184 F F F
167892023-09-22T23:31:45.629Z 10 200-219 125 125 125 190 190 190 F F F
167902023-09-22T23:31:45.632Z 11 220-239 119 119 119 179 179 179 F F F
167912023-09-22T23:31:45.635Z 12 240-259 126 126 126 191 191 191 F F F
167922023-09-22T23:31:45.638Z 13 260-279 125 125 125 190 190 190 F F F
167932023-09-22T23:31:45.641Z 14 280-299 122 122 122 184 184 184 F F F
167942023-09-22T23:31:45.644Z 15 300-319 121 121 121 183 183 183 F F F
167952023-09-22T23:31:45.647Z 16 320-339 126 126 126 191 191 191 F F F
167962023-09-22T23:31:45.650Z 17 340-359 126 126 126 191 191 191 F F F
167972023-09-22T23:31:45.653Z 18 360-379 123 123 123 186 186 186 F F F
167982023-09-22T23:31:45.655Z 19 380-399 121 121 121 183 183 183 F F F
167992023-09-22T23:31:45.659Z 20 400-419 118 118 118 176 176 176 F F F
168002023-09-22T23:31:45.661Z 21 420-439 124 124 124 189 189 189 F F F
168012023-09-22T23:31:45.664Z 22 440-459 126 126 126 191 191 191 F F F
168022023-09-22T23:31:45.673Z 23 460-479 126 126 126 191 191 191 F F F
168032023-09-22T23:31:45.681Z 24 480-499 121 121 121 183 183 183 F F F
168042023-09-22T23:31:45.696Z 25 500-519 125 125 125 190 190 190 F F F
168052023-09-22T23:31:45.702Z 26 520-539 125 125 125 190 190 190 F F F
168062023-09-22T23:31:45.709Z 27 540-559 118 118 118 177 177 177 F F F
168072023-09-22T23:31:45.714Z 28 560-579 127 127 127 192 192 192 F F F
168082023-09-22T23:31:45.717Z 29 580-599 125 125 125 190 190 190 F F F
168092023-09-22T23:31:45.720ZMax gen: 127, Max flush: 192
168102023-09-22T23:31:45.724Z
168112023-09-22T23:31:45.727ZBegin loop 64
168122023-09-22T23:31:45.730ZDownstairs to restart: 1
168132023-09-22T23:31:45.734Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.635353694Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168142023-09-22T23:31:45.737Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.635732893Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168152023-09-22T23:31:45.741Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.635796007Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168162023-09-22T23:31:45.744Z{"msg":"Crucible stats registered with UUID: 09958677-23f1-4d45-879a-cd32345400e9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.635864078Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168172023-09-22T23:31:45.747Z{"msg":"Crucible 09958677-23f1-4d45-879a-cd32345400e9 has session id: 5626897c-e926-4ba8-aa41-c90e50777ab1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.636015392Z","hostname":"Crucible runtime is spawned
168182023-09-22T23:31:45.750Zip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168192023-09-22T23:31:45.753Z{The guest has requested activation with gen:128
168202023-09-22T23:31:45.756Z"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.636272874Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689,"looper":"0"}
168212023-09-22T23:31:45.760Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.636352612Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689,"task":"up_listen"}
168222023-09-22T23:31:45.763Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.636462746Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168232023-09-22T23:31:45.766Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.636508427Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168242023-09-22T23:31:45.769Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.636550668Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689,"looper":"2"}
168252023-09-22T23:31:45.772Z{"msg":"[0] 09958677-23f1-4d45-879a-cd32345400e9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.636604769Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689,"looper":"0"}
168262023-09-22T23:31:45.774Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.636673553Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689,"looper":"1"}
168272023-09-22T23:31:45.778Z{"msg":"[2] 09958677-23f1-4d45-879a-cd32345400e9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.636742149Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689,"looper":"2"}
168282023-09-22T23:31:45.781Z{"msg":"09958677-23f1-4d45-879a-cd32345400e9 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.637305921Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168292023-09-22T23:31:45.784Z{"msg":"Set desired generation to :128","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.637360636Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168302023-09-22T23:31:45.787Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.63740313Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168312023-09-22T23:31:45.790Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.637717652Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168322023-09-22T23:31:45.793Z{"msg":"[0] 09958677-23f1-4d45-879a-cd32345400e9 (bad5cf75-6847-4a97-9030-bdfbcba07dc1) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.638061115Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168332023-09-22T23:31:45.796Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.638230282Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168342023-09-22T23:31:45.799Z{"msg":"[0] client is_active_req TRUE, promote! session bad5cf75-6847-4a97-9030-bdfbcba07dc1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.638335895Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168352023-09-22T23:31:45.802Z{"msg":"[2] 09958677-23f1-4d45-879a-cd32345400e9 (bad5cf75-6847-4a97-9030-bdfbcba07dc1) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.638453188Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168362023-09-22T23:31:45.805Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.638519255Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168372023-09-22T23:31:45.808Z{"msg":"[2] client is_active_req TRUE, promote! session bad5cf75-6847-4a97-9030-bdfbcba07dc1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.638580948Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168382023-09-22T23:31:45.810Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.638831188Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168392023-09-22T23:31:45.814Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.638893424Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168402023-09-22T23:31:45.818Z{"msg":"09958677-23f1-4d45-879a-cd32345400e9 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.639026171Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168412023-09-22T23:31:45.821Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.639101092Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168422023-09-22T23:31:45.824Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.639196121Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168432023-09-22T23:31:45.827Z{"msg":"09958677-23f1-4d45-879a-cd32345400e9 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.639246681Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168442023-09-22T23:31:45.829Z{"msg":"[0] 09958677-23f1-4d45-879a-cd32345400e9 (bad5cf75-6847-4a97-9030-bdfbcba07dc1) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.641101115Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168452023-09-22T23:31:45.834Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.641156963Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168462023-09-22T23:31:45.837Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:31:45.641179047Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168472023-09-22T23:31:45.840Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.641335504Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168482023-09-22T23:31:45.844Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.641408634Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168492023-09-22T23:31:45.849Z{"msg":"09958677-23f1-4d45-879a-cd32345400e9 WaitQuorum New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.641458061Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168502023-09-22T23:31:45.856Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.641497749Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168512023-09-22T23:31:45.862Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.641542465Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168522023-09-22T23:31:45.870Z{"msg":"[2] 09958677-23f1-4d45-879a-cd32345400e9 (bad5cf75-6847-4a97-9030-bdfbcba07dc1) WaitQuorum New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.641590633Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168532023-09-22T23:31:45.877Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.641631862Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168542023-09-22T23:31:45.884Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:31:45.641673116Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168552023-09-22T23:31:45.888Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.64169303Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168562023-09-22T23:31:45.891Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.64173826Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168572023-09-22T23:31:45.894Z{"msg":"09958677-23f1-4d45-879a-cd32345400e9 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.641783816Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168582023-09-22T23:31:45.897Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.641836581Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168592023-09-22T23:31:45.899Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:45.641856379Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168602023-09-22T23:31:46.642Z{"msg":"[1] 09958677-23f1-4d45-879a-cd32345400e9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.638779421Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689,"looper":"1"}
168612023-09-22T23:31:46.647Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.638862706Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168622023-09-22T23:31:46.655Z{"msg":"[1] 09958677-23f1-4d45-879a-cd32345400e9 (bad5cf75-6847-4a97-9030-bdfbcba07dc1) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.640033175Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168632023-09-22T23:31:46.661Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.640089901Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168642023-09-22T23:31:46.669Z{"msg":"[1] client is_active_req TRUE, promote! session bad5cf75-6847-4a97-9030-bdfbcba07dc1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.640133062Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168652023-09-22T23:31:46.674Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.640721652Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168662023-09-22T23:31:46.678Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.640802501Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168672023-09-22T23:31:46.684Z{"msg":"09958677-23f1-4d45-879a-cd32345400e9 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.640867774Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168682023-09-22T23:31:46.690Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.642013407Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168692023-09-22T23:31:46.695Z{"msg":"[1] 09958677-23f1-4d45-879a-cd32345400e9 (bad5cf75-6847-4a97-9030-bdfbcba07dc1) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.643186092Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168702023-09-22T23:31:46.701Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.643242268Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168712023-09-22T23:31:46.706Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:31:46.643289614Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168722023-09-22T23:31:46.712Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.643328703Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168732023-09-22T23:31:46.717Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.643367777Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168742023-09-22T23:31:46.724Z{"msg":"09958677-23f1-4d45-879a-cd32345400e9 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.64340791Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168752023-09-22T23:31:46.728Z{"msg":"[0]R flush_numbers[0..12]: [183, 190, 185, 192, 182, 191, 188, 191, 191, 184, 190, 179]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.643448791Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168762023-09-22T23:31:46.732Z{"msg":"[0]R generation[0..12]: [121, 125, 122, 127, 120, 126, 124, 126, 126, 122, 125, 119]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.643488776Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168772023-09-22T23:31:46.739Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.643534823Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168782023-09-22T23:31:46.744Z{"msg":"[1]R flush_numbers[0..12]: [183, 190, 185, 192, 182, 191, 188, 191, 191, 184, 190, 179]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.643574439Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168792023-09-22T23:31:46.751Z{"msg":"[1]R generation[0..12]: [121, 125, 122, 127, 120, 126, 124, 126, 126, 122, 125, 119]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:128
168802023-09-22T23:31:46.761Z,"time":"2023-09-22T23:31:46.643684249Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal"Wait for a query_work_queue command to finish before sending IO
168812023-09-22T23:31:46.769Z,"pid":1689}
168822023-09-22T23:31:46.773Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.643732298Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168832023-09-22T23:31:46.777Z{"msg":"[2]R flush_numbers[0..12]: [183, 190, 185, 192, 182, 191, 188, 191, 191, 184, 190, 179]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.64380548Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168842023-09-22T23:31:46.785Z{"msg":"[2]R generation[0..12]: [121, 125, 122, 127, 120, 126, 124, 126, 126, 122, 125, 119]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.643876052Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168852023-09-22T23:31:46.789Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.643949838Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168862023-09-22T23:31:46.802Z{"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.644014228Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168872023-09-22T23:31:46.807Z{"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.64411696Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168882023-09-22T23:31:46.822Z{"msg":"Next flush: 193","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.644174548Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168892023-09-22T23:31:46.834Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.644223052Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168902023-09-22T23:31:46.849Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.644244429Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168912023-09-22T23:31:46.860Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.644284334Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168922023-09-22T23:31:46.873Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.644323516Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168932023-09-22T23:31:46.884Z{"msg":"09958677-23f1-4d45-879a-cd32345400e9 is now active with session: bad5cf75-6847-4a97-9030-bdfbcba07dc1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.644369756Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168942023-09-22T23:31:46.891Z{"msg":"09958677-23f1-4d45-879a-cd32345400e9 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.644391524Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168952023-09-22T23:31:46.902Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.644429278Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168962023-09-22T23:31:46.907Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.644448289Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168972023-09-22T23:31:46.929Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.644484443Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168982023-09-22T23:31:46.940Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.644503738Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
168992023-09-22T23:31:46.951ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
169002023-09-22T23:31:46.971Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.644543565Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
169012023-09-22T23:31:46.977Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.644562643Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
169022023-09-22T23:31:46.980Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.644598699Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
169032023-09-22T23:31:46.985Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.644635719Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
169042023-09-22T23:31:46.991Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.644673088Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
169052023-09-22T23:31:46.995Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.644710382Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
169062023-09-22T23:31:47.001Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.644747533Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1689}
169072023-09-22T23:31:47.006ZLoading write count information from file "/tmp/test_repair_verify.data"
169082023-09-22T23:31:47.009ZRun Repair workload
169092023-09-22T23:31:47.012Z01/30 Write block 159 len 32768 data: 14 13 13 13 11 10 10 8
169102023-09-22T23:31:47.015Z02/30 Read block 42 len 40960
169112023-09-22T23:31:47.018Z03/30 Write block 102 len 24576 data: 14 15 18 20 18 17
169122023-09-22T23:31:47.021Z04/30 Read block 538 len 36864
169132023-09-22T23:31:47.024Z05/30 Read block 550 len 28672
169142023-09-22T23:31:47.027Z06/30 Read block 179 len 4096
169152023-09-22T23:31:47.030Z07/30 Read block 376 len 16384
169162023-09-22T23:31:47.034Z08/30 Read block 125 len 40960
169172023-09-22T23:31:47.037Z09/30 Read block 563 len 20480
169182023-09-22T23:31:47.041Z10/30 Flush
169192023-09-22T23:31:47.044Z11/30 Write block 180 len 8192 data: 10 11
169202023-09-22T23:31:47.047Z12/30 Read block 82 len 12288
169212023-09-22T23:31:47.050Z13/30 Write block 208 len 24576 data: 17 14 15 17 13 12
169222023-09-22T23:31:47.053Z14/30 Write block 292 len 28672 data: 16 13 12 13 12 9 9
169232023-09-22T23:31:47.055Z15/30 Read block 497 len 16384
169242023-09-22T23:31:47.058Z16/30 Write block 103 len 32768 data: 16 19 21 19 18 18 16 16
169252023-09-22T23:31:47.062Z17/30 Flush
169262023-09-22T23:31:47.065Z18/30 Write block 342 len 20480 data: 10 8 9 11 12
169272023-09-22T23:31:47.068Z19/30 Read block 296 len 24576
169282023-09-22T23:31:47.071Z20/30 Flush
169292023-09-22T23:31:47.074Z21/30 Write block 73 len 36864 data: 8 9 10 10 12 12 15 13 14
169302023-09-22T23:31:47.077Z22/30 Read block 285 len 16384
169312023-09-22T23:31:47.080Z23/30 Write block 395 len 40960 data: 13 12 11 12 10 13 10 13 11 11
169322023-09-22T23:31:47.083Z24/30 Read block 20 len 4096
169332023-09-22T23:31:47.086Z25/30 Write block 269 len 20480 data: 14 14 17 18 15
169342023-09-22T23:31:47.089Z26/30 Read block 416 len 20480
169352023-09-22T23:31:47.092Z27/30 Write block 484 len 28672 data: 13 13 16 13 17 18 15
169362023-09-22T23:31:47.096Z28/30 Write block 323 len 28672 data: 10 8 5 7 6 10 9
169372023-09-22T23:31:47.099Z29/30 Read block 119 len 20480
169382023-09-22T23:31:47.103Z30/30 Read block 489 len 28672
169392023-09-22T23:31:47.105Z----------------------------------------------------------------
169402023-09-22T23:31:47.110Z Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:30
169412023-09-22T23:31:47.113ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
169422023-09-22T23:31:47.117Z 1 Acked 1000 Write 8 Done Done Done false
169432023-09-22T23:31:47.120Z 2 Acked 1001 Read 10 Done Done Done false
169442023-09-22T23:31:47.128Z 3 Acked 1002 Write 6 Done Sent Done false
169452023-09-22T23:31:47.131Z 4 Acked 1003 Read 9 Done Sent Done false
169462023-09-22T23:31:47.138Z 5 Acked 1004 Read 7 Done Sent Done false
169472023-09-22T23:31:47.142Z 6 Acked 1005 Read 1 Done Sent Done false
169482023-09-22T23:31:47.148Z 7 Acked 1006 Read 4 Done Sent Done false
169492023-09-22T23:31:47.559Z 8 Acked 1007 Read 10 Done Sent Done false
169502023-09-22T23:31:47.563Z 9 Acked 1008 Read 5 Done Sent Done false
169512023-09-22T23:31:47.566Z 10 Acked 1009 Flush 0 Done Sent Done false
169522023-09-22T23:31:47.569Z 11 Acked 1010 Write 2 Done Sent Done false
169532023-09-22T23:31:47.573Z 12 Acked 1011 Read 3 Done Sent Done false
169542023-09-22T23:31:47.577Z 13 Acked 1012 Write 6 Done Sent Done false
169552023-09-22T23:31:47.581Z 14 Acked 1013 Write 7 Done Sent Done false
169562023-09-22T23:31:47.585Z 15 Acked 1014 Read 4 Done Sent Done false
169572023-09-22T23:31:47.588Z 16 Acked 1015 Write 8 Done Sent Done false
169582023-09-22T23:31:47.592ZError: Difference in extent metadata found!
169592023-09-22T23:31:47.595Z 17 Acked 1016 Flush 0 Done Sent Done false
169602023-09-22T23:31:47.598Z 18 Acked 1017 Write 5 Done Sent Done false
169612023-09-22T23:31:47.601Z 19 Acked 1018 Read 6 Done Sent Done false
169622023-09-22T23:31:47.604Z 20 Acked 1019 Flush 0 Done Sent Done false
169632023-09-22T23:31:47.608Z 21 Acked 1020 Write 9 Done Sent Done false
169642023-09-22T23:31:47.611Z 22 Acked 1021 Read 4 Done Sent Done false
169652023-09-22T23:31:47.614Z 23 Acked 1022 Write 10 Done Sent Done false
169662023-09-22T23:31:47.617Z 24 Acked 1023 Read 1 Done Sent Done false
169672023-09-22T23:31:47.621Z 25 Acked 1024 Write 5 Done Sent Done false
169682023-09-22T23:31:47.624Z 26 Acked 1025 Read 5 Done Sent Done false
169692023-09-22T23:31:47.628Z 27 Acked 1026 Write 7 Done Sent Done false
169702023-09-22T23:31:47.631Z 28 Acked 1027 Write 7 Done Sent Done false
169712023-09-22T23:31:47.634Z 29 Acked 1028 Read 5 Done Sent Done false
169722023-09-22T23:31:47.637Z 30 Acked 1029 Read 7 Sent Sent Done false
169732023-09-22T23:31:47.640Z STATES DS:0 DS:1 DS:2 TOTAL
169742023-09-22T23:31:47.643Z New 0 0 0 0
169752023-09-22T23:31:47.647Z Sent 1 28 0 29
169762023-09-22T23:31:47.649Z Done 29 2 30 61
169772023-09-22T23:31:47.652Z Skipped 0 0 0 0
169782023-09-22T23:31:47.655Z Error 0 0 0 0
169792023-09-22T23:31:47.659ZLast Flush: 1019 0 1019
169802023-09-22T23:31:47.662ZDownstairs last five completed:
169812023-09-22T23:31:47.665ZUpstairs last five completed: 30 29 28 27 26 25
169822023-09-22T23:31:47.667ZWrote out file "/tmp/test_repair_verify.data"
169832023-09-22T23:31:47.671Z
169842023-09-22T23:31:47.674ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
169852023-09-22T23:31:47.677Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.794449443Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1690}
169862023-09-22T23:31:47.680Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.794747913Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1690}
169872023-09-22T23:31:47.683Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.794770941Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1690}
169882023-09-22T23:31:47.686Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.825492877Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1690}
169892023-09-22T23:31:47.688Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.825528511Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1690}
169902023-09-22T23:31:47.692Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.825547958Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1690}
169912023-09-22T23:31:47.694Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.849758569Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1690}
169922023-09-22T23:31:47.697Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.849791244Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1690}
169932023-09-22T23:31:47.701Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:46.849810293Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1690}
169942023-09-22T23:31:47.704ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
169952023-09-22T23:31:47.708Z 0 000-019 121 121 121 183 183 183 F F F
169962023-09-22T23:31:47.712Z 1 020-039 125 125 125 190 190 190 F F F
169972023-09-22T23:31:47.715Z 2 040-059 122 122 122 185 185 185 F F F
169982023-09-22T23:31:47.718Z 3 060-079 127 127 127 192 192 192 T F T <---
169992023-09-22T23:31:47.720Z 4 080-099 120 120 120 182 182 182 T F T <---
170002023-09-22T23:31:47.723Z 5 100-119 128 126 128 194 191 194 F F F <---
170012023-09-22T23:31:47.726Z 6 120-139 124 124 124 188 188 188 F F F
170022023-09-22T23:31:47.730Z 7 140-159 128 126 128 193 191 193 F T F <---
170032023-09-22T23:31:47.733Z 8 160-179 128 126 128 193 191 193 F T F <---
170042023-09-22T23:31:47.736Z 9 180-199 128 122 128 194 184 194 F F F <---
170052023-09-22T23:31:47.740Z 10 200-219 128 125 128 194 190 194 F F F <---
170062023-09-22T23:31:47.747Z 11 220-239 119 119 119 179 179 179 F F F
170072023-09-22T23:31:47.750Z 12 240-259 126 126 126 191 191 191 F F F
170082023-09-22T23:31:47.752Z 13 260-279 125 125 125 190 190 190 T F T <---
170092023-09-22T23:31:47.755Z 14 280-299 128 122 128 194 184 194 F F F <---
170102023-09-22T23:31:47.758Z 15 300-319 121 121 121 183 183 183 F F F
170112023-09-22T23:31:47.761Z 16 320-339 126 126 126 191 191 191 T F T <---
170122023-09-22T23:31:47.763Z 17 340-359 128 126 128 195 191 195 F F F <---
170132023-09-22T23:31:47.766Z 18 360-379 123 123 123 186 186 186 F F F
170142023-09-22T23:31:47.770Z 19 380-399 121 121 121 183 183 183 T F T <---
170152023-09-22T23:31:47.775Z 20 400-419 118 118 118 176 176 176 T F T <---
170162023-09-22T23:31:48.187Z 21 420-439 124 124 124 189 189 189 F F F
170172023-09-22T23:31:48.192Z 22 440-459 126 126 126 191 191 191 F F F
170182023-09-22T23:31:48.195Z 23 460-479 126 126 126 191 191 191 F F F
170192023-09-22T23:31:48.198Z 24 480-499 121 121 121 183 183 183 T F T <---
170202023-09-22T23:31:48.201Z 25 500-519 125 125 125 190 190 190 F F F
170212023-09-22T23:31:48.204Z 26 520-539 125 125 125 190 190 190 F F F
170222023-09-22T23:31:48.207Z 27 540-559 118 118 118 177 177 177 F F F
170232023-09-22T23:31:48.209Z 28 560-579 127 127 127 192 192 192 F F F
170242023-09-22T23:31:48.213Z 29 580-599 125 125 125 190 190 190 F F F
170252023-09-22T23:31:48.216ZMax gen: 128, Max flush: 195
170262023-09-22T23:31:48.219ZA Difference in extent metadata is expected here
170272023-09-22T23:31:48.222ZOn loop 64
170282023-09-22T23:31:48.225Z
170292023-09-22T23:31:48.227ZVerifying data now
170302023-09-22T23:31:53.373ZNo change after verify
170312023-09-22T23:31:53.379ZLoop: 64 Downstairs dump after verify (and repair):
170322023-09-22T23:31:53.394Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.389897224Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1695}
170332023-09-22T23:31:53.403Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.390290435Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1695}
170342023-09-22T23:31:53.413Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.390315387Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1695}
170352023-09-22T23:31:53.430Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.409043542Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1695}
170362023-09-22T23:31:53.437Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.409096328Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1695}
170372023-09-22T23:31:53.443Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.409134528Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1695}
170382023-09-22T23:31:53.449Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.420126488Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1695}
170392023-09-22T23:31:53.455Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.420178037Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1695}
170402023-09-22T23:31:53.464Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.420216573Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1695}
170412023-09-22T23:31:53.477ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
170422023-09-22T23:31:53.490Z 0 000-019 121 121 121 183 183 183 F F F
170432023-09-22T23:31:53.502Z 1 020-039 125 125 125 190 190 190 F F F
170442023-09-22T23:31:53.513Z 2 040-059 122 122 122 185 185 185 F F F
170452023-09-22T23:31:54.539Z 3 060-079 129 129 129 196 196 196 F F F
170462023-09-22T23:31:54.559Z 4 080-099 129 129 129 196 196 196 F F F
170472023-09-22T23:31:54.572Z 5 100-119 128 128 128 194 194 194 F F F
170482023-09-22T23:31:54.582Z 6 120-139 124 124 124 188 188 188 F F F
170492023-09-22T23:31:54.585Z 7 140-159 128 128 128 193 193 193 F F F
170502023-09-22T23:31:54.589Z 8 160-179 128 128 128 193 193 193 F F F
170512023-09-22T23:31:54.592Z 9 180-199 128 128 128 194 194 194 F F F
170522023-09-22T23:31:54.595Z 10 200-219 128 128 128 194 194 194 F F F
170532023-09-22T23:31:54.598Z 11 220-239 119 119 119 179 179 179 F F F
170542023-09-22T23:31:54.602Z 12 240-259 126 126 126 191 191 191 F F F
170552023-09-22T23:31:54.605Z 13 260-279 129 129 129 196 196 196 F F F
170562023-09-22T23:31:54.608Z 14 280-299 128 128 128 194 194 194 F F F
170572023-09-22T23:31:54.612Z 15 300-319 121 121 121 183 183 183 F F F
170582023-09-22T23:31:54.616Z 16 320-339 129 129 129 196 196 196 F F F
170592023-09-22T23:31:54.622Z 17 340-359 128 128 128 195 195 195 F F F
170602023-09-22T23:31:54.625Z 18 360-379 123 123 123 186 186 186 F F F
170612023-09-22T23:31:54.628Z 19 380-399 129 129 129 196 196 196 F F F
170622023-09-22T23:31:54.631Z 20 400-419 129 129 129 196 196 196 F F F
170632023-09-22T23:31:54.634Z 21 420-439 124 124 124 189 189 189 F F F
170642023-09-22T23:31:54.637Z 22 440-459 126 126 126 191 191 191 F F F
170652023-09-22T23:31:54.640Z 23 460-479 126 126 126 191 191 191 F F F
170662023-09-22T23:31:54.643Z 24 480-499 129 129 129 196 196 196 F F F
170672023-09-22T23:31:54.646Z 25 500-519 125 125 125 190 190 190 F F F
170682023-09-22T23:31:54.649Z 26 520-539 125 125 125 190 190 190 F F F
170692023-09-22T23:31:54.652Z 27 540-559 118 118 118 177 177 177 F F F
170702023-09-22T23:31:54.655Z 28 560-579 127 127 127 192 192 192 F F F
170712023-09-22T23:31:54.658Z 29 580-599 125 125 125 190 190 190 F F F
170722023-09-22T23:31:54.661ZMax gen: 129, Max flush: 196
170732023-09-22T23:31:54.664Z
170742023-09-22T23:31:54.668ZBegin loop 65
170752023-09-22T23:31:54.671ZDownstairs to restart: 2
170762023-09-22T23:31:54.674Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.470270445Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
170772023-09-22T23:31:54.697Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.470675223Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
170782023-09-22T23:31:54.700Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.470707228Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
170792023-09-22T23:31:54.704Z{"msg":"Crucible stats registered with UUID: 612705fa-20a0-4c79-9ac8-d043f40a730d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.470764395Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
170802023-09-22T23:31:54.707Z{"msg":"Crucible 612705fa-20a0-4c79-9ac8-d043f40a730d has session id: 2ffed678-e62e-4c38-b17d-bfb5537c49f5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.470788558Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
170812023-09-22T23:31:54.710ZCrucible runtime is spawned
170822023-09-22T23:31:54.714Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30The guest has requested activation with gen:130
170832023-09-22T23:31:54.717Z,"time":"2023-09-22T23:31:53.471136621Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697,"task":"up_listen"}
170842023-09-22T23:31:54.720Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.471334519Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
170852023-09-22T23:31:54.723Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.471382926Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
170862023-09-22T23:31:54.726Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.471431123Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697,"looper":"2"}
170872023-09-22T23:31:54.729Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.471480986Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697,"looper":"1"}
170882023-09-22T23:31:54.732Z{"msg":"[1] 612705fa-20a0-4c79-9ac8-d043f40a730d looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.471693881Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697,"looper":"1"}
170892023-09-22T23:31:54.735Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.471770981Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697,"looper":"0"}
170902023-09-22T23:31:54.738Z{"msg":"[0] 612705fa-20a0-4c79-9ac8-d043f40a730d looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.471965453Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697,"looper":"0"}
170912023-09-22T23:31:54.741Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.472036414Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
170922023-09-22T23:31:54.746Z{"msg":"612705fa-20a0-4c79-9ac8-d043f40a730d active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.472099475Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
170932023-09-22T23:31:54.749Z{"msg":"Set desired generation to :130","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.472144711Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
170942023-09-22T23:31:54.754Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.472554907Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
170952023-09-22T23:31:54.757Z{"msg":"[1] 612705fa-20a0-4c79-9ac8-d043f40a730d (60e2872c-524f-4ede-8c7e-014087e71f0e) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.472744157Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
170962023-09-22T23:31:54.760Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.472814949Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
170972023-09-22T23:31:54.763Z{"msg":"[1] client is_active_req TRUE, promote! session 60e2872c-524f-4ede-8c7e-014087e71f0e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.472860501Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
170982023-09-22T23:31:54.766Z{"msg":"[0] 612705fa-20a0-4c79-9ac8-d043f40a730d (60e2872c-524f-4ede-8c7e-014087e71f0e) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.472902098Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
170992023-09-22T23:31:54.769Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.472940562Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171002023-09-22T23:31:54.772Z{"msg":"[0] client is_active_req TRUE, promote! session 60e2872c-524f-4ede-8c7e-014087e71f0e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.473046815Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171012023-09-22T23:31:54.775Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.473488451Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171022023-09-22T23:31:54.778Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.473570174Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171032023-09-22T23:31:54.781Z{"msg":"612705fa-20a0-4c79-9ac8-d043f40a730d WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.473645461Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171042023-09-22T23:31:54.784Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.473723868Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171052023-09-22T23:31:54.787Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.473804016Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171062023-09-22T23:31:54.790Z{"msg":"612705fa-20a0-4c79-9ac8-d043f40a730d WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.47387732Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171072023-09-22T23:31:54.793Z{"msg":"[1] 612705fa-20a0-4c79-9ac8-d043f40a730d (60e2872c-524f-4ede-8c7e-014087e71f0e) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.475822408Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171082023-09-22T23:31:54.796Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.475870803Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171092023-09-22T23:31:54.799Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:31:53.475896185Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171102023-09-22T23:31:54.802Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.475913805Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171112023-09-22T23:31:54.805Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.47594931Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171122023-09-22T23:31:54.808Z{"msg":"612705fa-20a0-4c79-9ac8-d043f40a730d WaitActive WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.475969404Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171132023-09-22T23:31:54.811Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.476002219Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171142023-09-22T23:31:54.814Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.476021755Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171152023-09-22T23:31:54.817Z{"msg":"[0] 612705fa-20a0-4c79-9ac8-d043f40a730d (60e2872c-524f-4ede-8c7e-014087e71f0e) WaitActive WaitQuorum New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.476143603Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171162023-09-22T23:31:54.820Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.476170294Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171172023-09-22T23:31:54.824Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:31:53.476188465Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171182023-09-22T23:31:54.827Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.476223314Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171192023-09-22T23:31:54.829Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.476243024Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171202023-09-22T23:31:54.832Z{"msg":"612705fa-20a0-4c79-9ac8-d043f40a730d WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.476261656Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171212023-09-22T23:31:54.835Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.476279274Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171222023-09-22T23:31:54.838Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:53.476296783Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171232023-09-22T23:31:54.841Z{"msg":"[2] 612705fa-20a0-4c79-9ac8-d043f40a730d looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.472333426Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697,"looper":"2"}
171242023-09-22T23:31:54.844Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.472391886Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171252023-09-22T23:31:54.848Z{"msg":"[2] 612705fa-20a0-4c79-9ac8-d043f40a730d (60e2872c-524f-4ede-8c7e-014087e71f0e) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.473530809Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171262023-09-22T23:31:54.851Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.47355832Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171272023-09-22T23:31:54.854Z{"msg":"[2] client is_active_req TRUE, promote! session 60e2872c-524f-4ede-8c7e-014087e71f0e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.473578588Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171282023-09-22T23:31:54.858Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.474251546Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171292023-09-22T23:31:54.862Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.474284222Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171302023-09-22T23:31:54.865Z{"msg":"612705fa-20a0-4c79-9ac8-d043f40a730d WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.474305621Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171312023-09-22T23:31:54.868Z{"msg":"[2] 612705fa-20a0-4c79-9ac8-d043f40a730d (60e2872c-524f-4ede-8c7e-014087e71f0e) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.476673874Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171322023-09-22T23:31:54.871Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.476703717Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171332023-09-22T23:31:54.874Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:31:54.476722254Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171342023-09-22T23:31:54.877Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.476739834Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171352023-09-22T23:31:54.880Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.476757794Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171362023-09-22T23:31:54.883Z{"msg":"612705fa-20a0-4c79-9ac8-d043f40a730d WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.476776703Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171372023-09-22T23:31:54.886Z{"msg":"[0]R flush_numbers[0..12]: [183, 190, 185, 196, 196, 194, 188, 193, 193, 194, 194, 179]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.476805942Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171382023-09-22T23:31:54.889Z{"msg":"[0]R generation[0..12]: [121, 125, 122, 129, 129, 128, 124, 128, 128, 128, 128, 119]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.476825634Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171392023-09-22T23:31:54.897Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.476845447Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171402023-09-22T23:31:54.904Z{"msg":"[1]R flush_numbers[0..12]: [183, 190, 185, 196, 196, 194, 188, 193, 193, 194, 194, 179]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.47686458Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171412023-09-22T23:31:54.907Z{"msg":"[1]R generation[0..12]: [121, 125, 122, 129, 129, 128, 124, 128, 128, 128, 128, 119]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.476883769Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171422023-09-22T23:31:54.910Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.476903412Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171432023-09-22T23:31:54.913ZError: Difference in extent metadata found!
171442023-09-22T23:31:54.916Z{"msg":"[2]R flush_numbers[0..12]: [183, 190, 185, 196, 196, 194, 188, 193, 193, 194, 194, 179]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.476922172Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171452023-09-22T23:31:54.918Z{"msg":"[2]R generation[0..12]: [121, 125, 122, 129, 129, 128, 124, 128, 128, 128, 128, 119]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.476941123Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171462023-09-22T23:31:54.921Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.476960805Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171472023-09-22T23:31:54.924Z{"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.476978157Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171482023-09-22T23:31:54.927Z{"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.476995635Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171492023-09-22T23:31:54.930Z{"msg":"Next flush: 197","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.477012737Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171502023-09-22T23:31:54.933Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.477030175Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171512023-09-22T23:31:54.936Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.477047361Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171522023-09-22T23:31:54.939ZThe guest has finished waiting for activation with:130
171532023-09-22T23:31:54.943Z{"msg":"No initial repair work was required","v":0,"Wait for a query_work_queue command to finish before sending IO
171542023-09-22T23:31:54.946Zname":"crucible","level":30,"time":"2023-09-22T23:31:54.477070765Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171552023-09-22T23:31:54.949Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.477089053Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171562023-09-22T23:31:54.952Z{"msg":"612705fa-20a0-4c79-9ac8-d043f40a730d is now active with session: 60e2872c-524f-4ede-8c7e-014087e71f0e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.477108138Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171572023-09-22T23:31:54.955Z{"msg":"612705fa-20a0-4c79-9ac8-d043f40a730d Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.477126359Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171582023-09-22T23:31:54.959Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.477145456Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171592023-09-22T23:31:54.968Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.477162913Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171602023-09-22T23:31:55.386Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.4771897Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171612023-09-22T23:31:55.392Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.477257535Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171622023-09-22T23:31:55.396Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.477291511Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171632023-09-22T23:31:55.400Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.477326243Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171642023-09-22T23:31:55.403Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.477358956Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171652023-09-22T23:31:55.410Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.477392177Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171662023-09-22T23:31:55.413Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.477424258Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171672023-09-22T23:31:55.417Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.477458323Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171682023-09-22T23:31:55.421Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.477493089Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1697}
171692023-09-22T23:31:55.427ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
171702023-09-22T23:31:55.435ZLoading write count information from file "/tmp/test_repair_verify.data"
171712023-09-22T23:31:55.453ZRun Repair workload
171722023-09-22T23:31:55.482Z01/30 Write block 39 len 8192 data: 12 12
171732023-09-22T23:31:55.493Z02/30 Read block 195 len 20480
171742023-09-22T23:31:55.500Z03/30 Read block 66 len 28672
171752023-09-22T23:31:55.503Z04/30 Write block 586 len 8192 data: 6 6
171762023-09-22T23:31:55.507Z05/30 Write block 142 len 4096 data: 10
171772023-09-22T23:31:55.510Z06/30 Flush
171782023-09-22T23:31:55.513Z07/30 Write block 505 len 32768 data: 10 12 11 8 12 10 9 12
171792023-09-22T23:31:55.517Z08/30 Write block 87 len 20480 data: 9 9 9 8 10
171802023-09-22T23:31:55.521Z09/30 Read block 196 len 8192
171812023-09-22T23:31:55.524Z10/30 Read block 490 len 28672
171822023-09-22T23:31:55.527Z11/30 Flush
171832023-09-22T23:31:55.531Z12/30 Write block 456 len 12288 data: 7 9 9
171842023-09-22T23:31:55.535Z13/30 Write block 489 len 36864 data: 19 16 16 14 14 12 14 14 14
171852023-09-22T23:31:55.539Z14/30 Flush
171862023-09-22T23:31:55.542Z15/30 Write block 453 len 12288 data: 11 9 8
171872023-09-22T23:31:55.549Z16/30 Read block 428 len 12288
171882023-09-22T23:31:55.552Z17/30 Read block 304 len 16384
171892023-09-22T23:31:55.559Z18/30 Write block 305 len 4096 data: 10
171902023-09-22T23:31:55.565Z19/30 Write block 435 len 20480 data: 12 11 11 11 9
171912023-09-22T23:31:55.568Z20/30 Read block 56 len 36864
171922023-09-22T23:31:55.571Z21/30 Write block 112 len 36864 data: 12 12 10 9 9 10 9 9 10
171932023-09-22T23:31:55.574Z22/30 Write block 201 len 40960 data: 9 11 13 12 18 16 15 18 15 16
171942023-09-22T23:31:55.577Z23/30 Read block 434 len 32768
171952023-09-22T23:31:55.579Z24/30 Write block 435 len 20480 data: 13 12 12 12 10
171962023-09-22T23:31:55.582Z25/30 Read block 116 len 28672
171972023-09-22T23:31:55.585Z26/30 Write block 491 len 12288 data: 17 15 15
171982023-09-22T23:31:55.588Z27/30 Read block 393 len 16384
171992023-09-22T23:31:55.591Z28/30 Read block 28 len 12288
172002023-09-22T23:31:55.594Z29/30 Write block 301 len 12288 data: 9 11 12
172012023-09-22T23:31:55.597Z30/30 Read block 75 len 4096
172022023-09-22T23:31:55.600Z----------------------------------------------------------------
172032023-09-22T23:31:55.603Z Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:30
172042023-09-22T23:31:55.607ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
172052023-09-22T23:31:55.610Z 1 Acked 1000 Write 2 Done Done Done false
172062023-09-22T23:31:55.613Z 2 Acked 1001 Read 5 Done Done Done false
172072023-09-22T23:31:55.615Z 3 Acked 1002 Read 7 Done Done Sent false
172082023-09-22T23:31:55.619Z 4 Acked 1003 Write 2 Done Done Sent false
172092023-09-22T23:31:55.622Z 5 Acked 1004 Write 1 Done Done Sent false
172102023-09-22T23:31:55.625Z 6 Acked 1005 Flush 0 Done Done Sent false
172112023-09-22T23:31:55.628Z 7 Acked 1006 Write 8 Done Done Sent false
172122023-09-22T23:31:55.630Z 8 Acked 1007 Write 5 Done Done Sent false
172132023-09-22T23:31:55.633Z 9 Acked 1008 Read 2 Done Done Sent false
172142023-09-22T23:31:55.637Z 10 Acked 1009 Read 7 Done Done Sent false
172152023-09-22T23:31:55.640Z 11 Acked 1010 Flush 0 Done Done Sent false
172162023-09-22T23:31:55.643Z 12 Acked 1011 Write 3 Done Done Sent false
172172023-09-22T23:31:55.646Z 13 Acked 1012 Write 9 Done Done Sent false
172182023-09-22T23:31:55.649Z 14 Acked 1013 Flush 0 Done Done Sent false
172192023-09-22T23:31:55.652Z 15 Acked 1014 Write 3 Done Done Sent false
172202023-09-22T23:31:55.655Z 16 Acked 1015 Read 3 Done Done Sent false
172212023-09-22T23:31:55.658Z 17 Acked 1016 Read 4 Done Done Sent false
172222023-09-22T23:31:55.662Z 18 Acked 1017 Write 1 Done Done Sent false
172232023-09-22T23:31:55.665Z 19 Acked 1018 Write 5 Done Done Sent false
172242023-09-22T23:31:55.668Z 20 Acked 1019 Read 9 Done Done Sent false
172252023-09-22T23:31:55.671Z 21 Acked 1020 Write 9 Done Done Sent false
172262023-09-22T23:31:55.674Z 22 Acked 1021 Write 10 Done Done Sent false
172272023-09-22T23:31:55.678Z 23 Acked 1022 Read 8 Done Done Sent false
172282023-09-22T23:31:55.681Z 24 Acked 1023 Write 5 Done Done Sent false
172292023-09-22T23:31:55.684Z 25 Acked 1024 Read 7 Done Done Sent false
172302023-09-22T23:31:55.687Z 26 Acked 1025 Write 3 Done Done Sent false
172312023-09-22T23:31:55.690Z 27 Acked 1026 Read 4 Done Done Sent false
172322023-09-22T23:31:55.693Z 28 Acked 1027 Read 3 Done Done Sent false
172332023-09-22T23:31:55.696Z 29 Acked 1028 Write 3 Done Done Sent false
172342023-09-22T23:31:55.719Z 30 Acked 1029 Read 1 Done Sent Sent false
172352023-09-22T23:31:55.722Z STATES DS:0 DS:1 DS:2 TOTAL
172362023-09-22T23:31:55.725Z New 0 0 0 0
172372023-09-22T23:31:55.729Z Sent 0 1 28 29
172382023-09-22T23:31:55.732Z Done 30 29 2 61
172392023-09-22T23:31:55.735Z Skipped 0 0 0 0
172402023-09-22T23:31:55.738Z Error 0 0 0 0
172412023-09-22T23:31:55.741ZLast Flush: 1013 1013 0
172422023-09-22T23:31:55.744ZDownstairs last five completed:
172432023-09-22T23:31:55.748ZUpstairs last five completed: 30 29 28 27 26 25
172442023-09-22T23:31:55.751ZWrote out file "/tmp/test_repair_verify.data"
172452023-09-22T23:31:55.754Z
172462023-09-22T23:31:55.757ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
172472023-09-22T23:31:55.760Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.618337242Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1698}
172482023-09-22T23:31:55.764Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.618611849Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1698}
172492023-09-22T23:31:55.767Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.618634608Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1698}
172502023-09-22T23:31:55.769Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.649676884Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1698}
172512023-09-22T23:31:55.772Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.649726458Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1698}
172522023-09-22T23:31:55.777Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.64974552Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1698}
172532023-09-22T23:31:55.780Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.67052804Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1698}
172542023-09-22T23:31:55.783Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.670568785Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1698}
172552023-09-22T23:31:55.786Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:31:54.670588283Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1698}
172562023-09-22T23:31:55.789ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
172572023-09-22T23:31:55.792Z 0 000-019 121 121 121 183 183 183 F F F
172582023-09-22T23:31:55.795Z 1 020-039 130 130 125 197 197 190 F F T <---
172592023-09-22T23:31:55.798Z 2 040-059 130 130 122 197 197 185 F F T <---
172602023-09-22T23:31:55.801Z 3 060-079 129 129 129 196 196 196 F F F
172612023-09-22T23:31:55.804Z 4 080-099 130 130 129 198 198 196 F F F <---
172622023-09-22T23:31:55.807Z 5 100-119 128 128 128 194 194 194 T T F <---
172632023-09-22T23:31:55.811Z 6 120-139 124 124 124 188 188 188 T T F <---
172642023-09-22T23:31:55.814Z 7 140-159 130 130 128 197 197 193 F F F <---
172652023-09-22T23:31:55.817Z 8 160-179 128 128 128 193 193 193 F F F
172662023-09-22T23:31:55.820Z 9 180-199 128 128 128 194 194 194 F F F
172672023-09-22T23:31:55.823Z 10 200-219 128 128 128 194 194 194 T T F <---
172682023-09-22T23:31:55.826Z 11 220-239 119 119 119 179 179 179 F F F
172692023-09-22T23:31:55.829Z 12 240-259 126 126 126 191 191 191 F F F
172702023-09-22T23:31:55.832Z 13 260-279 129 129 129 196 196 196 F F F
172712023-09-22T23:31:55.835Z 14 280-299 128 128 128 194 194 194 F F F
172722023-09-22T23:31:55.841Z 15 300-319 121 121 121 183 183 183 T T F <---
172732023-09-22T23:31:55.844Z 16 320-339 129 129 129 196 196 196 F F F
172742023-09-22T23:31:55.847Z 17 340-359 128 128 128 195 195 195 F F F
172752023-09-22T23:31:55.850Z 18 360-379 123 123 123 186 186 186 F F F
172762023-09-22T23:31:55.853Z 19 380-399 129 129 129 196 196 196 F F F
172772023-09-22T23:31:55.856Z 20 400-419 129 129 129 196 196 196 F F F
172782023-09-22T23:31:55.859Z 21 420-439 124 124 124 189 189 189 T T F <---
172792023-09-22T23:31:55.862Z 22 440-459 130 130 126 199 199 191 T T F <---
172802023-09-22T23:31:55.865Z 23 460-479 126 126 126 191 191 191 F F F
172812023-09-22T23:31:55.868Z 24 480-499 130 130 129 199 199 196 T T F <---
172822023-09-22T23:31:55.871Z 25 500-519 130 130 125 198 198 190 F F F <---
172832023-09-22T23:31:55.874Z 26 520-539 125 125 125 190 190 190 F F F
172842023-09-22T23:31:55.877Z 27 540-559 118 118 118 177 177 177 F F F
172852023-09-22T23:31:55.880Z 28 560-579 127 127 127 192 192 192 F F F
172862023-09-22T23:31:55.886Z 29 580-599 130 130 125 197 197 190 F F F <---
172872023-09-22T23:31:55.894ZMax gen: 130, Max flush: 199
172882023-09-22T23:31:55.904ZA Difference in extent metadata is expected here
172892023-09-22T23:31:55.925ZOn loop 65
172902023-09-22T23:31:56.346Z
172912023-09-22T23:31:56.349ZVerifying data now
172922023-09-22T23:32:01.155ZNo change after verify
172932023-09-22T23:32:01.159ZLoop: 65 Downstairs dump after verify (and repair):
172942023-09-22T23:32:01.164Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.164382956Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1703}
172952023-09-22T23:32:01.168Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.16469682Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1703}
172962023-09-22T23:32:01.173Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.164722417Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1703}
172972023-09-22T23:32:01.185Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.184144507Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1703}
172982023-09-22T23:32:01.191Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.184201935Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1703}
172992023-09-22T23:32:01.194Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.184223278Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1703}
173002023-09-22T23:32:01.199Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.195275701Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1703}
173012023-09-22T23:32:01.202Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.195327785Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1703}
173022023-09-22T23:32:01.205Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.195365727Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1703}
173032023-09-22T23:32:01.210ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
173042023-09-22T23:32:01.220Z 0 000-019 121 121 121 183 183 183 F F F
173052023-09-22T23:32:01.642Z 1 020-039 130 130 130 197 197 197 F F F
173062023-09-22T23:32:01.650Z 2 040-059 130 130 130 197 197 197 F F F
173072023-09-22T23:32:01.657Z 3 060-079 129 129 129 196 196 196 F F F
173082023-09-22T23:32:01.680Z 4 080-099 130 130 130 198 198 198 F F F
173092023-09-22T23:32:01.700Z 5 100-119 131 131 131 200 200 200 F F F
173102023-09-22T23:32:01.717Z 6 120-139 131 131 131 200 200 200 F F F
173112023-09-22T23:32:01.727Z 7 140-159 130 130 130 197 197 197 F F F
173122023-09-22T23:32:01.736Z 8 160-179 128 128 128 193 193 193 F F F
173132023-09-22T23:32:01.741Z 9 180-199 128 128 128 194 194 194 F F F
173142023-09-22T23:32:01.746Z 10 200-219 131 131 131 200 200 200 F F F
173152023-09-22T23:32:01.751Z 11 220-239 119 119 119 179 179 179 F F F
173162023-09-22T23:32:01.756Z 12 240-259 126 126 126 191 191 191 F F F
173172023-09-22T23:32:01.760Z 13 260-279 129 129 129 196 196 196 F F F
173182023-09-22T23:32:01.764Z 14 280-299 128 128 128 194 194 194 F F F
173192023-09-22T23:32:01.769Z 15 300-319 131 131 131 200 200 200 F F F
173202023-09-22T23:32:01.774Z 16 320-339 129 129 129 196 196 196 F F F
173212023-09-22T23:32:01.777Z 17 340-359 128 128 128 195 195 195 F F F
173222023-09-22T23:32:01.780Z 18 360-379 123 123 123 186 186 186 F F F
173232023-09-22T23:32:01.784Z 19 380-399 129 129 129 196 196 196 F F F
173242023-09-22T23:32:01.787Z 20 400-419 129 129 129 196 196 196 F F F
173252023-09-22T23:32:01.791Z 21 420-439 131 131 131 200 200 200 F F F
173262023-09-22T23:32:01.796Z 22 440-459 131 131 131 200 200 200 F F F
173272023-09-22T23:32:01.800Z 23 460-479 126 126 126 191 191 191 F F F
173282023-09-22T23:32:01.803Z 24 480-499 131 131 131 200 200 200 F F F
173292023-09-22T23:32:01.807Z 25 500-519 130 130 130 198 198 198 F F F
173302023-09-22T23:32:01.810Z 26 520-539 125 125 125 190 190 190 F F F
173312023-09-22T23:32:01.815Z 27 540-559 118 118 118 177 177 177 F F F
173322023-09-22T23:32:01.818Z 28 560-579 127 127 127 192 192 192 F F F
173332023-09-22T23:32:01.822Z 29 580-599 130 130 130 197 197 197 F F F
173342023-09-22T23:32:01.827ZMax gen: 131, Max flush: 200
173352023-09-22T23:32:01.831Z
173362023-09-22T23:32:01.856ZBegin loop 66
173372023-09-22T23:32:01.867ZDownstairs to restart: 2
173382023-09-22T23:32:01.871Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.244915232Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173392023-09-22T23:32:01.875Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.245284654Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173402023-09-22T23:32:01.880Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.245361912Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173412023-09-22T23:32:01.884Z{"msg":"Crucible stats registered with UUID: 77509d35-27f7-4f2b-898b-bfc33258de36","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.245434518Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173422023-09-22T23:32:01.887ZCrucible runtime is spawned
173432023-09-22T23:32:01.891Z{"msg":"Crucible 77509d35-27f7-4f2b-898b-bfc33258de36 has session id: dc778862-fac1-4c03-ae29-f36284c4167e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.245553923Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173442023-09-22T23:32:01.894Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30The guest has requested activation with gen:132
173452023-09-22T23:32:01.897Z,"time":"2023-09-22T23:32:01.245700455Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705,"task":"up_listen"}
173462023-09-22T23:32:01.900Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.245837121Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173472023-09-22T23:32:01.905Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.245920085Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173482023-09-22T23:32:01.916Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.245997409Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705,"looper":"1"}
173492023-09-22T23:32:01.939Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.246117802Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705,"looper":"2"}
173502023-09-22T23:32:01.958Z{"msg":"[1] 77509d35-27f7-4f2b-898b-bfc33258de36 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.246196902Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705,"looper":"1"}
173512023-09-22T23:32:02.377Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.246277361Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705,"looper":"0"}
173522023-09-22T23:32:02.396Z{"msg":"[0] 77509d35-27f7-4f2b-898b-bfc33258de36 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.246498255Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705,"looper":"0"}
173532023-09-22T23:32:02.412Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.246564789Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173542023-09-22T23:32:02.423Z{"msg":"77509d35-27f7-4f2b-898b-bfc33258de36 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.246625373Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173552023-09-22T23:32:02.436Z{"msg":"Set desired generation to :132","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.246734093Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173562023-09-22T23:32:02.444Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.247042734Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173572023-09-22T23:32:02.452Z{"msg":"[1] 77509d35-27f7-4f2b-898b-bfc33258de36 (6fec9184-e362-4c6c-a4a6-e551f72c5a13) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.247330053Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173582023-09-22T23:32:02.461Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.247387101Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173592023-09-22T23:32:02.472Z{"msg":"[1] client is_active_req TRUE, promote! session 6fec9184-e362-4c6c-a4a6-e551f72c5a13","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.247445211Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173602023-09-22T23:32:02.482Z{"msg":"[0] 77509d35-27f7-4f2b-898b-bfc33258de36 (6fec9184-e362-4c6c-a4a6-e551f72c5a13) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.24750294Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173612023-09-22T23:32:02.494Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.247562718Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173622023-09-22T23:32:02.508Z{"msg":"[0] client is_active_req TRUE, promote! session 6fec9184-e362-4c6c-a4a6-e551f72c5a13","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.24762604Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173632023-09-22T23:32:02.526Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.248015252Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173642023-09-22T23:32:02.540Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.248079714Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173652023-09-22T23:32:02.549Z{"msg":"77509d35-27f7-4f2b-898b-bfc33258de36 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.248122293Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173662023-09-22T23:32:02.561Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.248215394Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173672023-09-22T23:32:02.575Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.248285644Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173682023-09-22T23:32:02.586Z{"msg":"77509d35-27f7-4f2b-898b-bfc33258de36 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.248359092Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173692023-09-22T23:32:02.596Z{"msg":"[1] 77509d35-27f7-4f2b-898b-bfc33258de36 (6fec9184-e362-4c6c-a4a6-e551f72c5a13) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.250367236Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173702023-09-22T23:32:02.603Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.250436605Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173712023-09-22T23:32:02.611Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:01.250460905Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173722023-09-22T23:32:02.619Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.250479068Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173732023-09-22T23:32:02.627Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.250569065Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173742023-09-22T23:32:02.637Z{"msg":"77509d35-27f7-4f2b-898b-bfc33258de36 WaitActive WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.250594718Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173752023-09-22T23:32:03.058Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.250613388Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173762023-09-22T23:32:03.061Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.250630882Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173772023-09-22T23:32:03.064Z{"msg":"[0] 77509d35-27f7-4f2b-898b-bfc33258de36 (6fec9184-e362-4c6c-a4a6-e551f72c5a13) WaitActive WaitQuorum New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.25077976Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173782023-09-22T23:32:03.068Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.250808971Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173792023-09-22T23:32:03.071Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:01.250827337Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173802023-09-22T23:32:03.074Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.250844812Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173812023-09-22T23:32:03.078Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.250863125Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173822023-09-22T23:32:03.081Z{"msg":"77509d35-27f7-4f2b-898b-bfc33258de36 WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.250881576Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173832023-09-22T23:32:03.085Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.250899433Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173842023-09-22T23:32:03.088Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:01.250916738Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173852023-09-22T23:32:03.092Z{"msg":"[2] 77509d35-27f7-4f2b-898b-bfc33258de36 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.247796794Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705,"looper":"2"}
173862023-09-22T23:32:03.095Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.247974679Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173872023-09-22T23:32:03.098Z{"msg":"[2] 77509d35-27f7-4f2b-898b-bfc33258de36 (6fec9184-e362-4c6c-a4a6-e551f72c5a13) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.249025596Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173882023-09-22T23:32:03.101Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.249077928Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173892023-09-22T23:32:03.104Z{"msg":"[2] client is_active_req TRUE, promote! session 6fec9184-e362-4c6c-a4a6-e551f72c5a13","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.249121039Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173902023-09-22T23:32:03.108Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.249697915Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173912023-09-22T23:32:03.112Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.249757444Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173922023-09-22T23:32:03.115Z{"msg":"77509d35-27f7-4f2b-898b-bfc33258de36 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.249801268Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173932023-09-22T23:32:03.118Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.252100271Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173942023-09-22T23:32:03.124Z{"msg":"[2] 77509d35-27f7-4f2b-898b-bfc33258de36 (6fec9184-e362-4c6c-a4a6-e551f72c5a13) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.252260498Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173952023-09-22T23:32:03.535Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.252323996Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173962023-09-22T23:32:03.538Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:02.252379788Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173972023-09-22T23:32:03.542Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.252433003Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173982023-09-22T23:32:03.545Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.252487018Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
173992023-09-22T23:32:03.548Z{"msg":"77509d35-27f7-4f2b-898b-bfc33258de36 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.252542442Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174002023-09-22T23:32:03.551Z{"msg":"[0]R flush_numbers[0..12]: [183, 197, 197, 196, 198, 200, 200, 197, 193, 194, 200, 179]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:132
174012023-09-22T23:32:03.554ZWait for a query_work_queue command to finish before sending IO
174022023-09-22T23:32:03.558Z,"time":"2023-09-22T23:32:02.252598363Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174032023-09-22T23:32:03.561Z{"msg":"[0]R generation[0..12]: [121, 130, 130, 129, 130, 131, 131, 130, 128, 128, 131, 119]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.252771139Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174042023-09-22T23:32:03.565Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.252856824Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174052023-09-22T23:32:03.569Z{"msg":"[1]R flush_numbers[0..12]: [183, 197, 197, 196, 198, 200, 200, 197, 193, 194, 200, 179]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.252912937Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174062023-09-22T23:32:03.572Z{"msg":"[1]R generation[0..12]: [121, 130, 130, 129, 130, 131, 131, 130, 128, 128, 131, 119]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.25295534Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174072023-09-22T23:32:03.575Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.252998283Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174082023-09-22T23:32:03.579Z{"msg":"[2]R flush_numbers[0..12]: [183, 197, 197, 196, 198, 200, 200, 197, 193, 194, 200, 179]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.253038298Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174092023-09-22T23:32:03.583Z{"msg":"[2]R generation[0..12]: [121, 130, 130, 129, 130, 131, 131, 130, 128, 128, 131, 119]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.253079465Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174102023-09-22T23:32:03.586Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.253123594Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174112023-09-22T23:32:03.590Z{"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.253161455Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174122023-09-22T23:32:03.593Z{"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.253200467Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174132023-09-22T23:32:03.596Z{"msg":"Next flush: 201","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.253246833Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174142023-09-22T23:32:03.600Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.253289104Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174152023-09-22T23:32:03.604Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.253327972Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174162023-09-22T23:32:03.608Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.253371419Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174172023-09-22T23:32:03.611Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.253391304Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174182023-09-22T23:32:03.614Z{"msg":"77509d35-27f7-4f2b-898b-bfc33258de36 is now active with session: 6fec9184-e362-4c6c-a4a6-e551f72c5a13","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.253410437Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174192023-09-22T23:32:03.618Z{"msg":"77509d35-27f7-4f2b-898b-bfc33258de36 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.253428779Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174202023-09-22T23:32:03.621Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.253446642Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174212023-09-22T23:32:03.624Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.253464049Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174222023-09-22T23:32:03.628Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.253481469Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174232023-09-22T23:32:03.631Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.253498788Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174242023-09-22T23:32:03.634ZError: Difference in extent metadata found!
174252023-09-22T23:32:03.637Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.253515697Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174262023-09-22T23:32:03.640Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.253533069Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174272023-09-22T23:32:03.643Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.25355062Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174282023-09-22T23:32:03.646Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.253568195Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174292023-09-22T23:32:03.650Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.253585696Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174302023-09-22T23:32:03.653Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.253603143Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174312023-09-22T23:32:03.656Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.253620343Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1705}
174322023-09-22T23:32:03.660ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
174332023-09-22T23:32:03.663ZLoading write count information from file "/tmp/test_repair_verify.data"
174342023-09-22T23:32:03.666ZRun Repair workload
174352023-09-22T23:32:03.670Z01/30 Write block 469 len 28672 data: 12 9 7 8 7 7 6
174362023-09-22T23:32:03.673Z02/30 Read block 487 len 12288
174372023-09-22T23:32:03.677Z03/30 Write block 510 len 40960 data: 11 10 13 14 16 14 13 14 15 13
174382023-09-22T23:32:03.684Z04/30 Flush
174392023-09-22T23:32:03.700Z05/30 Write block 427 len 4096 data: 11
174402023-09-22T23:32:03.711Z06/30 Write block 403 len 40960 data: 12 12 10 9 9 8 10 8 10 8
174412023-09-22T23:32:03.721Z07/30 Read block 114 len 40960
174422023-09-22T23:32:03.732Z08/30 Write block 86 len 8192 data: 7 10
174432023-09-22T23:32:03.745Z09/30 Write block 546 len 4096 data: 12
174442023-09-22T23:32:03.752Z10/30 Write block 354 len 28672 data: 11 12 10 9 11 14 15
174452023-09-22T23:32:03.757Z11/30 Read block 81 len 24576
174462023-09-22T23:32:03.760Z12/30 Read block 245 len 40960
174472023-09-22T23:32:03.764Z13/30 Write block 242 len 36864 data: 11 8 7 8 5 6 6 7 6
174482023-09-22T23:32:03.767Z14/30 Read block 333 len 28672
174492023-09-22T23:32:03.770Z15/30 Read block 353 len 8192
174502023-09-22T23:32:03.773Z16/30 Read block 516 len 4096
174512023-09-22T23:32:03.776Z17/30 Write block 31 len 4096 data: 7
174522023-09-22T23:32:03.780Z18/30 Write block 139 len 32768 data: 10 10 10 11 10 6 7 6
174532023-09-22T23:32:03.784Z19/30 Read block 236 len 40960
174542023-09-22T23:32:03.787Z20/30 Flush
174552023-09-22T23:32:03.791Z21/30 Write block 575 len 4096 data: 6
174562023-09-22T23:32:03.794Z22/30 Read block 83 len 20480
174572023-09-22T23:32:03.798Z23/30 Write block 311 len 24576 data: 5 6 8 10 8 9
174582023-09-22T23:32:03.801Z24/30 Read block 139 len 4096
174592023-09-22T23:32:03.805Z25/30 Write block 172 len 8192 data: 12 13
174602023-09-22T23:32:03.807Z26/30 Write block 87 len 40960 data: 11 10 10 9 11 11 10 10 12 12
174612023-09-22T23:32:03.811Z27/30 Read block 462 len 12288
174622023-09-22T23:32:03.815Z28/30 Read block 40 len 12288
174632023-09-22T23:32:03.818Z29/30 Read block 403 len 40960
174642023-09-22T23:32:03.822Z30/30 Write block 290 len 28672 data: 10 12 17 14 13 14 13
174652023-09-22T23:32:03.825Z----------------------------------------------------------------
174662023-09-22T23:32:03.828Z Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30
174672023-09-22T23:32:03.831ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
174682023-09-22T23:32:03.835Z 1 Acked 1000 Write 7 Done Done Done false
174692023-09-22T23:32:03.838Z 2 Acked 1001 Read 3 Done Done Done false
174702023-09-22T23:32:03.842Z 3 Acked 1002 Write 10 Done Done Done false
174712023-09-22T23:32:03.845Z 4 Acked 1003 Flush 0 Done Done Sent false
174722023-09-22T23:32:03.848Z 5 Acked 1004 Write 1 Done Done Sent false
174732023-09-22T23:32:03.851Z 6 Acked 1005 Write 10 Done Done Sent false
174742023-09-22T23:32:03.854Z 7 Acked 1006 Read 10 Done Done Sent false
174752023-09-22T23:32:03.857Z 8 Acked 1007 Write 2 Done Done Sent false
174762023-09-22T23:32:03.861Z 9 Acked 1008 Write 1 Done Done Sent false
174772023-09-22T23:32:03.864Z 10 Acked 1009 Write 7 Done Done Sent false
174782023-09-22T23:32:03.867Z 11 Acked 1010 Read 6 Done Done Sent false
174792023-09-22T23:32:03.870Z 12 Acked 1011 Read 10 Done Done Sent false
174802023-09-22T23:32:03.874Z 13 Acked 1012 Write 9 Done Done Sent false
174812023-09-22T23:32:03.877Z 14 Acked 1013 Read 7 Done Done Sent false
174822023-09-22T23:32:03.880Z 15 Acked 1014 Read 2 Done Done Sent false
174832023-09-22T23:32:03.883Z 16 Acked 1015 Read 1 Done Done Sent false
174842023-09-22T23:32:03.886Z 17 Acked 1016 Write 1 Done Done Sent false
174852023-09-22T23:32:03.889Z 18 Acked 1017 Write 8 Done Done Sent false
174862023-09-22T23:32:03.913Z 19 Acked 1018 Read 10 Done Done Sent false
174872023-09-22T23:32:03.916Z 20 Acked 1019 Flush 0 Done Done Sent false
174882023-09-22T23:32:03.920Z 21 Acked 1020 Write 1 Done Done Sent false
174892023-09-22T23:32:03.923Z 22 Acked 1021 Read 5 Done Done Sent false
174902023-09-22T23:32:03.926Z 23 Acked 1022 Write 6 Done Done Sent false
174912023-09-22T23:32:03.930Z 24 Acked 1023 Read 1 Done Done Sent false
174922023-09-22T23:32:03.933Z 25 Acked 1024 Write 2 Done Done Sent false
174932023-09-22T23:32:03.937Z 26 Acked 1025 Write 10 Done Done Sent false
174942023-09-22T23:32:03.940Z 27 Acked 1026 Read 3 Done Done Sent false
174952023-09-22T23:32:03.943Z 28 Acked 1027 Read 3 Done Done Sent false
174962023-09-22T23:32:03.947Z 29 Acked 1028 Read 10 Done Done Sent false
174972023-09-22T23:32:03.950Z 30 Acked 1029 Write 7 Sent Sent Sent false
174982023-09-22T23:32:03.953Z STATES DS:0 DS:1 DS:2 TOTAL
174992023-09-22T23:32:03.956Z New 0 0 0 0
175002023-09-22T23:32:03.959Z Sent 1 1 27 29
175012023-09-22T23:32:03.962Z Done 29 29 3 61
175022023-09-22T23:32:03.967Z Skipped 0 0 0 0
175032023-09-22T23:32:03.970Z Error 0 0 0 0
175042023-09-22T23:32:03.974ZLast Flush: 1019 1019 0
175052023-09-22T23:32:03.977ZDownstairs last five completed:
175062023-09-22T23:32:03.980ZUpstairs last five completed: 30 29 28 27 26 25
175072023-09-22T23:32:03.983ZWrote out file "/tmp/test_repair_verify.data"
175082023-09-22T23:32:03.987Z
175092023-09-22T23:32:03.990ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
175102023-09-22T23:32:03.993Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.391520766Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1706}
175112023-09-22T23:32:03.996Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.391773534Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1706}
175122023-09-22T23:32:03.999Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.391795927Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1706}
175132023-09-22T23:32:04.003Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.419998035Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1706}
175142023-09-22T23:32:04.006Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.420030623Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1706}
175152023-09-22T23:32:04.009Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.420049485Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1706}
175162023-09-22T23:32:04.013Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.437791151Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1706}
175172023-09-22T23:32:04.016Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.4378205Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1706}
175182023-09-22T23:32:04.019Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:02.437840071Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1706}
175192023-09-22T23:32:04.022ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
175202023-09-22T23:32:04.025Z 0 000-019 121 121 121 183 183 183 F F F
175212023-09-22T23:32:04.028Z 1 020-039 132 132 130 202 202 197 F F F <---
175222023-09-22T23:32:04.031Z 2 040-059 130 130 130 197 197 197 F F F
175232023-09-22T23:32:04.033Z 3 060-079 129 129 129 196 196 196 F F F
175242023-09-22T23:32:04.036Z 4 080-099 132 132 130 202 202 198 T T F <---
175252023-09-22T23:32:04.040Z 5 100-119 131 131 131 200 200 200 F F F
175262023-09-22T23:32:04.044Z 6 120-139 132 132 131 202 202 200 F F F <---
175272023-09-22T23:32:04.047Z 7 140-159 132 132 130 202 202 197 F F F <---
175282023-09-22T23:32:04.050Z 8 160-179 128 128 128 193 193 193 T T F <---
175292023-09-22T23:32:04.053Z 9 180-199 128 128 128 194 194 194 F F F
175302023-09-22T23:32:04.056Z 10 200-219 131 131 131 200 200 200 F F F
175312023-09-22T23:32:04.059Z 11 220-239 119 119 119 179 179 179 F F F
175322023-09-22T23:32:04.061Z 12 240-259 132 132 126 202 202 191 F F F <---
175332023-09-22T23:32:04.064Z 13 260-279 129 129 129 196 196 196 F F F
175342023-09-22T23:32:04.067Z 14 280-299 128 128 128 194 194 194 T T F <---
175352023-09-22T23:32:04.070Z 15 300-319 131 131 131 200 200 200 T T F <---
175362023-09-22T23:32:04.073Z 16 320-339 129 129 129 196 196 196 F F F
175372023-09-22T23:32:04.076Z 17 340-359 132 132 128 202 202 195 F F F <---
175382023-09-22T23:32:04.079Z 18 360-379 132 132 123 202 202 186 F F F <---
175392023-09-22T23:32:04.083Z 19 380-399 129 129 129 196 196 196 F F F
175402023-09-22T23:32:04.086Z 20 400-419 132 132 129 202 202 196 F F F <---
175412023-09-22T23:32:04.089Z 21 420-439 132 132 131 202 202 200 F F F <---
175422023-09-22T23:32:04.093Z 22 440-459 131 131 131 200 200 200 F F F
175432023-09-22T23:32:04.096Z 23 460-479 132 132 126 201 201 191 F F T <---
175442023-09-22T23:32:04.100Z 24 480-499 131 131 131 200 200 200 F F F
175452023-09-22T23:32:04.104Z 25 500-519 132 132 130 201 201 198 F F T <---
175462023-09-22T23:32:04.111Z 26 520-539 125 125 125 190 190 190 F F F
175472023-09-22T23:32:04.115Z 27 540-559 132 132 118 202 202 177 F F F <---
175482023-09-22T23:32:04.118Z 28 560-579 127 127 127 192 192 192 T T F <---
175492023-09-22T23:32:04.121Z 29 580-599 130 130 130 197 197 197 F F F
175502023-09-22T23:32:04.124ZMax gen: 132, Max flush: 202
175512023-09-22T23:32:04.127ZA Difference in extent metadata is expected here
175522023-09-22T23:32:04.130ZOn loop 66
175532023-09-22T23:32:04.134Z
175542023-09-22T23:32:04.137ZVerifying data now
175552023-09-22T23:32:09.135ZNo change after verify
175562023-09-22T23:32:09.139ZLoop: 66 Downstairs dump after verify (and repair):
175572023-09-22T23:32:09.153Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.153257799Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1711}
175582023-09-22T23:32:09.157Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.153573093Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1711}
175592023-09-22T23:32:09.159Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.153598357Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1711}
175602023-09-22T23:32:09.180Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.172305837Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1711}
175612023-09-22T23:32:09.194Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.172359132Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1711}
175622023-09-22T23:32:09.208Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.172379557Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1711}
175632023-09-22T23:32:09.218Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.183270914Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1711}
175642023-09-22T23:32:09.229Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.183321752Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1711}
175652023-09-22T23:32:09.240Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.183360271Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1711}
175662023-09-22T23:32:09.250ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
175672023-09-22T23:32:09.260Z 0 000-019 121 121 121 183 183 183 F F F
175682023-09-22T23:32:09.281Z 1 020-039 132 132 132 202 202 202 F F F
175692023-09-22T23:32:09.295Z 2 040-059 130 130 130 197 197 197 F F F
175702023-09-22T23:32:09.319Z 3 060-079 129 129 129 196 196 196 F F F
175712023-09-22T23:32:09.331Z 4 080-099 133 133 133 203 203 203 F F F
175722023-09-22T23:32:09.347Z 5 100-119 131 131 131 200 200 200 F F F
175732023-09-22T23:32:09.363Z 6 120-139 132 132 132 202 202 202 F F F
175742023-09-22T23:32:09.372Z 7 140-159 132 132 132 202 202 202 F F F
175752023-09-22T23:32:09.383Z 8 160-179 133 133 133 203 203 203 F F F
175762023-09-22T23:32:09.394Z 9 180-199 128 128 128 194 194 194 F F F
175772023-09-22T23:32:09.404Z 10 200-219 131 131 131 200 200 200 F F F
175782023-09-22T23:32:09.414Z 11 220-239 119 119 119 179 179 179 F F F
175792023-09-22T23:32:09.421Z 12 240-259 132 132 132 202 202 202 F F F
175802023-09-22T23:32:09.431Z 13 260-279 129 129 129 196 196 196 F F F
175812023-09-22T23:32:09.437Z 14 280-299 133 133 133 203 203 203 F F F
175822023-09-22T23:32:09.440Z 15 300-319 133 133 133 203 203 203 F F F
175832023-09-22T23:32:09.443Z 16 320-339 129 129 129 196 196 196 F F F
175842023-09-22T23:32:09.446Z 17 340-359 132 132 132 202 202 202 F F F
175852023-09-22T23:32:09.449Z 18 360-379 132 132 132 202 202 202 F F F
175862023-09-22T23:32:09.452Z 19 380-399 129 129 129 196 196 196 F F F
175872023-09-22T23:32:09.456Z 20 400-419 132 132 132 202 202 202 F F F
175882023-09-22T23:32:09.459Z 21 420-439 132 132 132 202 202 202 F F F
175892023-09-22T23:32:09.462Z 22 440-459 131 131 131 200 200 200 F F F
175902023-09-22T23:32:09.466Z 23 460-479 132 132 132 201 201 201 F F F
175912023-09-22T23:32:09.469Z 24 480-499 131 131 131 200 200 200 F F F
175922023-09-22T23:32:09.473Z 25 500-519 132 132 132 201 201 201 F F F
175932023-09-22T23:32:09.475Z 26 520-539 125 125 125 190 190 190 F F F
175942023-09-22T23:32:09.478Z 27 540-559 132 132 132 202 202 202 F F F
175952023-09-22T23:32:09.482Z 28 560-579 133 133 133 203 203 203 F F F
175962023-09-22T23:32:09.485Z 29 580-599 130 130 130 197 197 197 F F F
175972023-09-22T23:32:09.490ZMax gen: 133, Max flush: 203
175982023-09-22T23:32:09.494Z
175992023-09-22T23:32:09.497ZBegin loop 67
176002023-09-22T23:32:09.500ZDownstairs to restart: 1
176012023-09-22T23:32:09.503Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.233350572Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176022023-09-22T23:32:09.506Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.233766775Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176032023-09-22T23:32:09.513Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.233819136Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176042023-09-22T23:32:09.517Z{"msg":"Crucible stats registered with UUID: 9693bf26-1ae9-4299-91ec-cf0e63c08bfa","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.23386944Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176052023-09-22T23:32:09.530Z{"msg":"Crucible 9693bf26-1ae9-4299-91ec-cf0e63c08bfa has session id: 8867d4d3-451b-4525-87fd-1dbec59ce210","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.233911452Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176062023-09-22T23:32:09.547ZCrucible runtime is spawned
176072023-09-22T23:32:09.560ZThe guest has requested activation with gen:134
176082023-09-22T23:32:09.572Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.234221132Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713,"task":"up_listen"}
176092023-09-22T23:32:09.585Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.234277716Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176102023-09-22T23:32:09.597Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.234318405Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176112023-09-22T23:32:09.608Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.234340024Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713,"looper":"1"}
176122023-09-22T23:32:09.619Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.234520335Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713,"looper":"0"}
176132023-09-22T23:32:09.632Z{"msg":"[0] 9693bf26-1ae9-4299-91ec-cf0e63c08bfa looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.234704095Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713,"looper":"0"}
176142023-09-22T23:32:09.654Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.234815216Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713,"looper":"2"}
176152023-09-22T23:32:09.669Z{"msg":"[2] 9693bf26-1ae9-4299-91ec-cf0e63c08bfa looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.235066722Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713,"looper":"2"}
176162023-09-22T23:32:09.673Z{"msg":"9693bf26-1ae9-4299-91ec-cf0e63c08bfa active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.23538894Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176172023-09-22T23:32:09.677Z{"msg":"Set desired generation to :134","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.235426526Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176182023-09-22T23:32:09.680Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.235646046Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176192023-09-22T23:32:09.683Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.236072512Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176202023-09-22T23:32:09.686Z{"msg":"[0] 9693bf26-1ae9-4299-91ec-cf0e63c08bfa (7faa2011-c5f3-429c-93d7-411a76a6be50) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.23640316Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176212023-09-22T23:32:09.689Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.236458917Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176222023-09-22T23:32:09.693Z{"msg":"[0] client is_active_req TRUE, promote! session 7faa2011-c5f3-429c-93d7-411a76a6be50","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.23648292Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176232023-09-22T23:32:09.697Z{"msg":"[2] 9693bf26-1ae9-4299-91ec-cf0e63c08bfa (7faa2011-c5f3-429c-93d7-411a76a6be50) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.236665235Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176242023-09-22T23:32:09.700Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.236746666Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176252023-09-22T23:32:09.703Z{"msg":"[2] client is_active_req TRUE, promote! session 7faa2011-c5f3-429c-93d7-411a76a6be50","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.236888287Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176262023-09-22T23:32:09.706Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.237054813Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176272023-09-22T23:32:09.709Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.23713477Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176282023-09-22T23:32:09.714Z{"msg":"9693bf26-1ae9-4299-91ec-cf0e63c08bfa WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.237206829Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176292023-09-22T23:32:09.717Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.237279754Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176302023-09-22T23:32:09.720Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.237359055Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176312023-09-22T23:32:09.723Z{"msg":"9693bf26-1ae9-4299-91ec-cf0e63c08bfa WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.237432515Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176322023-09-22T23:32:09.727Z{"msg":"[2] 9693bf26-1ae9-4299-91ec-cf0e63c08bfa (7faa2011-c5f3-429c-93d7-411a76a6be50) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.239477487Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176332023-09-22T23:32:09.730Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.239520799Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176342023-09-22T23:32:09.734Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:09.239541872Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176352023-09-22T23:32:09.739Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.239570419Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176362023-09-22T23:32:09.742Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.239674592Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176372023-09-22T23:32:09.745Z{"msg":"9693bf26-1ae9-4299-91ec-cf0e63c08bfa WaitActive New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.239702184Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176382023-09-22T23:32:09.748Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.239720649Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176392023-09-22T23:32:09.751Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.239738273Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176402023-09-22T23:32:09.754Z{"msg":"[0] 9693bf26-1ae9-4299-91ec-cf0e63c08bfa (7faa2011-c5f3-429c-93d7-411a76a6be50) WaitActive New WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.239859604Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176412023-09-22T23:32:09.758Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.239885525Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176422023-09-22T23:32:09.761Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:09.239903912Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176432023-09-22T23:32:09.764Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.239921457Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176442023-09-22T23:32:09.767Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.239939249Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176452023-09-22T23:32:09.770Z{"msg":"9693bf26-1ae9-4299-91ec-cf0e63c08bfa WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.239957698Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176462023-09-22T23:32:09.773Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.239975912Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176472023-09-22T23:32:09.778Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:09.239993308Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176482023-09-22T23:32:10.238Z{"msg":"[1] 9693bf26-1ae9-4299-91ec-cf0e63c08bfa looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.235975828Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713,"looper":"1"}
176492023-09-22T23:32:10.244Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.236059033Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176502023-09-22T23:32:10.247Z{"msg":"[1] 9693bf26-1ae9-4299-91ec-cf0e63c08bfa (7faa2011-c5f3-429c-93d7-411a76a6be50) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.237189617Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176512023-09-22T23:32:10.250Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.237246739Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176522023-09-22T23:32:10.253Z{"msg":"[1] client is_active_req TRUE, promote! session 7faa2011-c5f3-429c-93d7-411a76a6be50","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.23727055Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176532023-09-22T23:32:10.256Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.237886582Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176542023-09-22T23:32:10.259Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.237942701Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176552023-09-22T23:32:10.262Z{"msg":"9693bf26-1ae9-4299-91ec-cf0e63c08bfa WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.237985266Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176562023-09-22T23:32:10.265Z{"msg":"[1] 9693bf26-1ae9-4299-91ec-cf0e63c08bfa (7faa2011-c5f3-429c-93d7-411a76a6be50) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.240366359Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176572023-09-22T23:32:10.268Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.240442963Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176582023-09-22T23:32:10.271Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:10.240511457Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176592023-09-22T23:32:10.274Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.240559864Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176602023-09-22T23:32:10.281Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.240600379Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176612023-09-22T23:32:10.284Z{"msg":"9693bf26-1ae9-4299-91ec-cf0e63c08bfa WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.240640146Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176622023-09-22T23:32:10.287Z{"msg":"[0]R flush_numbers[0..12]: [183, 202, 197, 196, 203, 200, 202, 202, 203, 194, 200, 179]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.240683623Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176632023-09-22T23:32:10.290Z{"msg":"[0]R generation[0..12]: [121, 132, 130, 129, 133, 131, 132, 132, 133, 128, 131, 119]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.240724204Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176642023-09-22T23:32:10.293Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]"The guest has finished waiting for activation with:134
176652023-09-22T23:32:10.296Z,"v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
176662023-09-22T23:32:10.299Z,"time":"2023-09-22T23:32:10.240783639Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176672023-09-22T23:32:10.303Z{"msg":"[1]R flush_numbers[0..12]: [183, 202, 197, 196, 203, 200, 202, 202, 203, 194, 200, 179]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.240806021Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176682023-09-22T23:32:10.306Z{"msg":"[1]R generation[0..12]: [121, 132, 130, 129, 133, 131, 132, 132, 133, 128, 131, 119]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.240849058Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176692023-09-22T23:32:10.309Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.240890587Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176702023-09-22T23:32:10.312Z{"msg":"[2]R flush_numbers[0..12]: [183, 202, 197, 196, 203, 200, 202, 202, 203, 194, 200, 179]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.240938049Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176712023-09-22T23:32:10.315Z{"msg":"[2]R generation[0..12]: [121, 132, 130, 129, 133, 131, 132, 132, 133, 128, 131, 119]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.2409898Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176722023-09-22T23:32:10.318Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.241049049Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176732023-09-22T23:32:10.321Z{"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.241095099Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176742023-09-22T23:32:10.324Z{"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.241134525Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176752023-09-22T23:32:10.327Z{"msg":"Next flush: 204","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.241179065Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176762023-09-22T23:32:10.331Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.241217861Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176772023-09-22T23:32:10.334Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.241237515Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176782023-09-22T23:32:10.337Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.241254931Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176792023-09-22T23:32:10.340Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.241291699Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176802023-09-22T23:32:10.343Z{"msg":"9693bf26-1ae9-4299-91ec-cf0e63c08bfa is now active with session: 7faa2011-c5f3-429c-93d7-411a76a6be50","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.241331248Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176812023-09-22T23:32:10.346Z{"msg":"9693bf26-1ae9-4299-91ec-cf0e63c08bfa Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.241419602Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176822023-09-22T23:32:10.349Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.241440388Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176832023-09-22T23:32:10.352Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.241476482Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176842023-09-22T23:32:10.355Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.241519402Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176852023-09-22T23:32:10.358Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.241556722Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176862023-09-22T23:32:10.361Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.241595579Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176872023-09-22T23:32:10.365Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.241632647Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176882023-09-22T23:32:10.368Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
176892023-09-22T23:32:10.371Z,"time":"2023-09-22T23:32:10.24166971Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176902023-09-22T23:32:10.374Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.241781761Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176912023-09-22T23:32:10.377Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.241825741Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176922023-09-22T23:32:10.381Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.241863994Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176932023-09-22T23:32:10.384Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.24190107Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1713}
176942023-09-22T23:32:10.387ZLoading write count information from file "/tmp/test_repair_verify.data"
176952023-09-22T23:32:10.390ZRun Repair workload
176962023-09-22T23:32:10.393Z01/30 Write block 455 len 36864 data: 9 8 10 10 14 13 12 10 11
176972023-09-22T23:32:10.396Z02/30 Write block 372 len 12288 data: 8 7 8
176982023-09-22T23:32:10.399Z03/30 Read block 303 len 8192
176992023-09-22T23:32:10.402Z04/30 Write block 172 len 20480 data: 13 14 14 16 18
177002023-09-22T23:32:10.404Z05/30 Write block 128 len 40960 data: 10 8 9 8 12 11 10 9 10 10
177012023-09-22T23:32:10.408Z06/30 Read block 436 len 40960
177022023-09-22T23:32:10.411Z07/30 Write block 370 len 12288 data: 6 7 9
177032023-09-22T23:32:10.414Z08/30 Read block 346 len 16384
177042023-09-22T23:32:10.417Z09/30 Write block 228 len 40960 data: 9 8 9 5 6 5 6 7 8 8
177052023-09-22T23:32:10.420Z10/30 Flush
177062023-09-22T23:32:10.423Z11/30 Write block 334 len 16384 data: 12 13 12 9
177072023-09-22T23:32:10.426Z12/30 Read block 424 len 4096
177082023-09-22T23:32:10.429Z13/30 Write block 411 len 20480 data: 11 9 9 11 11
177092023-09-22T23:32:10.432Z14/30 Read block 6 len 16384
177102023-09-22T23:32:10.435Z15/30 Write block 350 len 20480 data: 8 10 10 10 12
177112023-09-22T23:32:10.438Z16/30 Write block 349 len 8192 data: 9 9
177122023-09-22T23:32:10.441Z17/30 Read block 587 len 40960
177132023-09-22T23:32:10.444Z18/30 Write block 60 len 40960 data: 13 13 15 18 17 20 21 20 21 19
177142023-09-22T23:32:10.447Z19/30 Write block 102 len 28672 data: 15 17 20 22 20 19 19
177152023-09-22T23:32:10.450Z20/30 Write block 453 len 24576 data: 12 10 10 9 11 11
177162023-09-22T23:32:10.453Z21/30 Write block 231 len 40960 data: 6 7 6 7 8 9 9 9 11 13
177172023-09-22T23:32:10.456Z22/30 Read block 273 len 24576
177182023-09-22T23:32:10.459Z23/30 Read block 4 len 36864
177192023-09-22T23:32:10.462Z24/30 Write block 495 len 12288 data: 15 15 15
177202023-09-22T23:32:10.465Z25/30 Write block 322 len 12288 data: 8 11 9
177212023-09-22T23:32:10.468Z26/30 Read block 535 len 40960
177222023-09-22T23:32:10.474Z27/30 Read block 158 len 8192
177232023-09-22T23:32:10.477Z28/30 Write block 418 len 8192 data: 12 13
177242023-09-22T23:32:10.480Z29/30 Write block 393 len 36864 data: 9 11 14 13 12 13 11 14 11
177252023-09-22T23:32:10.483Z30/30 Read block 551 len 16384
177262023-09-22T23:32:10.486Z----------------------------------------------------------------
177272023-09-22T23:32:10.489Z Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:20
177282023-09-22T23:32:10.492ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
177292023-09-22T23:32:10.495Z 11 Acked 1010 Write 4 Done Done Done false
177302023-09-22T23:32:10.498Z 12 Acked 1011 Read 1 Done Done Done false
177312023-09-22T23:32:10.501Z 13 Acked 1012 Write 5 Done Done Done false
177322023-09-22T23:32:10.504Z 14 Acked 1013 Read 4 Done Done Done false
177332023-09-22T23:32:10.508Z 15 Acked 1014 Write 5 Done Done Done false
177342023-09-22T23:32:10.511Z 16 Acked 1015 Write 2 Done Done Done false
177352023-09-22T23:32:10.514Z 17 Acked 1016 Read 10 Done Done Done false
177362023-09-22T23:32:10.517Z 18 Acked 1017 Write 10 Done Done Done false
177372023-09-22T23:32:10.519Z 19 Acked 1018 Write 7 Done Done Done false
177382023-09-22T23:32:10.522Z 20 Acked 1019 Write 6 Done Done Done false
177392023-09-22T23:32:10.525Z 21 Acked 1020 Write 10 Done Done Done false
177402023-09-22T23:32:10.528Z 22 Acked 1021 Read 6 Done Done Done false
177412023-09-22T23:32:10.532Z 23 Acked 1022 Read 9 Done Sent Done false
177422023-09-22T23:32:10.535Z 24 Acked 1023 Write 3 Done Sent Done false
177432023-09-22T23:32:10.538Z 25 Acked 1024 Write 3 Done Sent Done false
177442023-09-22T23:32:10.541Z 26 Acked 1025 Read 10 Done Sent Done false
177452023-09-22T23:32:10.544Z 27 Acked 1026 Read 2 Done Sent Done false
177462023-09-22T23:32:10.548Z 28 Acked 1027 Write 2 Done Sent Done false
177472023-09-22T23:32:10.551Z 29 Acked 1028 Write 9 Done Sent Done false
177482023-09-22T23:32:10.554Z 30 Acked 1029 Read 4 Done Sent Done false
177492023-09-22T23:32:10.557Z STATES DS:0 DS:1 DS:2 TOTAL
177502023-09-22T23:32:10.560Z New 0 0 0 0
177512023-09-22T23:32:10.563Z Sent 0 8 0 8
177522023-09-22T23:32:10.566Z Done 20 12 20 52
177532023-09-22T23:32:10.569Z Skipped 0 0 0 0
177542023-09-22T23:32:10.572Z Error 0 0 0 0
177552023-09-22T23:32:10.575ZLast Flush: 1009 1009 1009
177562023-09-22T23:32:10.578ZDownstairs last five completed: 1009 1008 1007 1006 1005 1004
177572023-09-22T23:32:10.581ZUpstairs last five completed: 30 29 28 27 26 25
177582023-09-22T23:32:10.584ZWrote out file "/tmp/test_repair_verify.data"
177592023-09-22T23:32:10.587Z
177602023-09-22T23:32:10.590ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
177612023-09-22T23:32:10.593Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.385898354Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1714}
177622023-09-22T23:32:10.597Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.386248447Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1714}
177632023-09-22T23:32:10.599Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.386270991Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1714}
177642023-09-22T23:32:10.602Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.421196589Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1714}
177652023-09-22T23:32:10.605Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.421249503Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1714}
177662023-09-22T23:32:10.608Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.421287877Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1714}
177672023-09-22T23:32:10.611Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.477430099Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1714}
177682023-09-22T23:32:10.614Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.477490683Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1714}
177692023-09-22T23:32:10.617Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:10.477513319Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1714}
177702023-09-22T23:32:10.620ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
177712023-09-22T23:32:10.623Z 0 000-019 121 121 121 183 183 183 F F F
177722023-09-22T23:32:10.626Z 1 020-039 132 132 132 202 202 202 F F F
177732023-09-22T23:32:10.629Z 2 040-059 130 130 130 197 197 197 F F F
177742023-09-22T23:32:10.632Z 3 060-079 129 129 129 196 196 196 T T T
177752023-09-22T23:32:10.635Z 4 080-099 133 133 133 203 203 203 F F F
177762023-09-22T23:32:10.638Z 5 100-119 131 131 131 200 200 200 T T T
177772023-09-22T23:32:10.641Z 6 120-139 134 134 134 204 204 204 F F F
177782023-09-22T23:32:10.644Z 7 140-159 132 132 132 202 202 202 F F F
177792023-09-22T23:32:10.647Z 8 160-179 134 134 134 204 204 204 F F F
177802023-09-22T23:32:10.650Z 9 180-199 128 128 128 194 194 194 F F F
177812023-09-22T23:32:10.653Z 10 200-219 131 131 131 200 200 200 F F F
177822023-09-22T23:32:10.656Z 11 220-239 134 134 134 204 204 204 T T T
177832023-09-22T23:32:10.659Z 12 240-259 132 132 132 202 202 202 T T T
177842023-09-22T23:32:10.662Z 13 260-279 129 129 129 196 196 196 F F F
177852023-09-22T23:32:10.666Z 14 280-299 133 133 133 203 203 203 F F F
177862023-09-22T23:32:10.669Z 15 300-319 133 133 133 203 203 203 F F F
177872023-09-22T23:32:10.672Z 16 320-339 129 129 129 196 196 196 T T T
177882023-09-22T23:32:10.675Z 17 340-359 132 132 132 202 202 202 T T T
177892023-09-22T23:32:10.678Z 18 360-379 134 134 134 204 204 204 F F F
177902023-09-22T23:32:10.681Z 19 380-399 129 129 129 196 196 196 T F T <---
177912023-09-22T23:32:10.684Z 20 400-419 132 132 132 202 202 202 T T T
177922023-09-22T23:32:10.688Z 21 420-439 132 132 132 202 202 202 F F F
177932023-09-22T23:32:10.690Z 22 440-459 134 134 134 204 204 204 T T T
177942023-09-22T23:32:10.693Z 23 460-479 134 134 134 204 204 204 F F F
177952023-09-22T23:32:10.696Z 24 480-499 131 131 131 200 200 200 T F T <---
177962023-09-22T23:32:10.699Z 25 500-519 132 132 132 201 201 201 F F F
177972023-09-22T23:32:10.702ZError: Difference in extent metadata found!
177982023-09-22T23:32:10.705Z 26 520-539 125 125 125 190 190 190 F F F
177992023-09-22T23:32:10.709Z 27 540-559 132 132 132 202 202 202 F F F
178002023-09-22T23:32:10.713Z 28 560-579 133 133 133 203 203 203 F F F
178012023-09-22T23:32:10.716Z 29 580-599 130 130 130 197 197 197 F F F
178022023-09-22T23:32:10.719ZMax gen: 134, Max flush: 204
178032023-09-22T23:32:10.722ZA Difference in extent metadata is expected here
178042023-09-22T23:32:10.726ZOn loop 67
178052023-09-22T23:32:10.729Z
178062023-09-22T23:32:10.731ZVerifying data now
178072023-09-22T23:32:16.701ZNo change after verify
178082023-09-22T23:32:16.704ZLoop: 67 Downstairs dump after verify (and repair):
178092023-09-22T23:32:16.719Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.71862263Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1719}
178102023-09-22T23:32:16.722Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.719025163Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1719}
178112023-09-22T23:32:16.725Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.719050864Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1719}
178122023-09-22T23:32:16.738Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.738032853Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1719}
178132023-09-22T23:32:16.741Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.738098571Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1719}
178142023-09-22T23:32:16.744Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.738120412Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1719}
178152023-09-22T23:32:16.749Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.749336455Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1719}
178162023-09-22T23:32:16.752Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.749388816Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1719}
178172023-09-22T23:32:16.755Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.749409425Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1719}
178182023-09-22T23:32:16.761ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
178192023-09-22T23:32:16.763Z 0 000-019 121 121 121 183 183 183 F F F
178202023-09-22T23:32:16.766Z 1 020-039 132 132 132 202 202 202 F F F
178212023-09-22T23:32:16.769Z 2 040-059 130 130 130 197 197 197 F F F
178222023-09-22T23:32:16.772Z 3 060-079 135 135 135 205 205 205 F F F
178232023-09-22T23:32:16.775Z 4 080-099 133 133 133 203 203 203 F F F
178242023-09-22T23:32:16.778Z 5 100-119 135 135 135 205 205 205 F F F
178252023-09-22T23:32:16.780Z 6 120-139 134 134 134 204 204 204 F F F
178262023-09-22T23:32:16.783Z 7 140-159 132 132 132 202 202 202 F F F
178272023-09-22T23:32:16.787Z 8 160-179 134 134 134 204 204 204 F F F
178282023-09-22T23:32:16.794Z 9 180-199 128 128 128 194 194 194 F F F
178292023-09-22T23:32:16.802Z 10 200-219 131 131 131 200 200 200 F F F
178302023-09-22T23:32:16.812Z 11 220-239 135 135 135 205 205 205 F F F
178312023-09-22T23:32:16.820Z 12 240-259 135 135 135 205 205 205 F F F
178322023-09-22T23:32:16.830Z 13 260-279 129 129 129 196 196 196 F F F
178332023-09-22T23:32:16.838Z 14 280-299 133 133 133 203 203 203 F F F
178342023-09-22T23:32:16.845Z 15 300-319 133 133 133 203 203 203 F F F
178352023-09-22T23:32:16.853Z 16 320-339 135 135 135 205 205 205 F F F
178362023-09-22T23:32:16.860Z 17 340-359 135 135 135 205 205 205 F F F
178372023-09-22T23:32:16.870Z 18 360-379 134 134 134 204 204 204 F F F
178382023-09-22T23:32:16.878Z 19 380-399 135 135 135 205 205 205 F F F
178392023-09-22T23:32:16.885Z 20 400-419 135 135 135 205 205 205 F F F
178402023-09-22T23:32:16.892Z 21 420-439 132 132 132 202 202 202 F F F
178412023-09-22T23:32:16.901Z 22 440-459 135 135 135 205 205 205 F F F
178422023-09-22T23:32:16.908Z 23 460-479 134 134 134 204 204 204 F F F
178432023-09-22T23:32:16.918Z 24 480-499 135 135 135 205 205 205 F F F
178442023-09-22T23:32:16.925Z 25 500-519 132 132 132 201 201 201 F F F
178452023-09-22T23:32:16.934Z 26 520-539 125 125 125 190 190 190 F F F
178462023-09-22T23:32:16.941Z 27 540-559 132 132 132 202 202 202 F F F
178472023-09-22T23:32:16.953Z 28 560-579 133 133 133 203 203 203 F F F
178482023-09-22T23:32:16.961Z 29 580-599 130 130 130 197 197 197 F F F
178492023-09-22T23:32:16.969ZMax gen: 135, Max flush: 205
178502023-09-22T23:32:16.975Z
178512023-09-22T23:32:16.984ZBegin loop 68
178522023-09-22T23:32:16.992ZDownstairs to restart: 2
178532023-09-22T23:32:17.002Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.798399448Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178542023-09-22T23:32:17.010Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.798804245Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178552023-09-22T23:32:17.018Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.798856694Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178562023-09-22T23:32:17.025Z{"msg":"Crucible stats registered with UUID: 97c01c1d-4ab1-4912-a462-acf8f364d48d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.798903618Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178572023-09-22T23:32:17.033Z{"msg":"Crucible 97c01c1d-4ab1-4912-a462-acf8f364d48d has session id: 4bfeb08e-2fce-45ed-bd45-e766d3e05b42","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.798946355Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178582023-09-22T23:32:17.040ZCrucible runtime is spawned
178592023-09-22T23:32:17.054ZThe guest has requested activation with gen:136
178602023-09-22T23:32:17.057Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.799332822Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721,"task":"up_listen"}
178612023-09-22T23:32:17.060Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.799389462Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178622023-09-22T23:32:17.063Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.799430123Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178632023-09-22T23:32:17.066Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.799451513Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721,"looper":"0"}
178642023-09-22T23:32:17.069Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.79981153Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721,"looper":"2"}
178652023-09-22T23:32:17.072Z{"msg":"[0] 97c01c1d-4ab1-4912-a462-acf8f364d48d looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.799926246Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721,"looper":"0"}
178662023-09-22T23:32:17.075Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.799983939Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721,"looper":"1"}
178672023-09-22T23:32:17.078Z{"msg":"[1] 97c01c1d-4ab1-4912-a462-acf8f364d48d looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.800123356Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721,"looper":"1"}
178682023-09-22T23:32:17.081Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.800182443Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178692023-09-22T23:32:17.084Z{"msg":"97c01c1d-4ab1-4912-a462-acf8f364d48d active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.800294046Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178702023-09-22T23:32:17.087Z{"msg":"Set desired generation to :136","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.800343687Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178712023-09-22T23:32:17.089Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.800724297Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178722023-09-22T23:32:17.092Z{"msg":"[0] 97c01c1d-4ab1-4912-a462-acf8f364d48d (4b61aad5-6735-4028-b97c-dbf6ca91d9bf) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.800944408Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178732023-09-22T23:32:17.095Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.801009719Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178742023-09-22T23:32:17.097Z{"msg":"[0] client is_active_req TRUE, promote! session 4b61aad5-6735-4028-b97c-dbf6ca91d9bf","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.801040475Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178752023-09-22T23:32:17.100Z{"msg":"[1] 97c01c1d-4ab1-4912-a462-acf8f364d48d (4b61aad5-6735-4028-b97c-dbf6ca91d9bf) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.801137088Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178762023-09-22T23:32:17.103Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.801179946Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178772023-09-22T23:32:17.105Z{"msg":"[1] client is_active_req TRUE, promote! session 4b61aad5-6735-4028-b97c-dbf6ca91d9bf","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.801240768Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178782023-09-22T23:32:17.108Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.801693123Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178792023-09-22T23:32:17.111Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.801777537Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178802023-09-22T23:32:17.114Z{"msg":"97c01c1d-4ab1-4912-a462-acf8f364d48d WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.801853249Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178812023-09-22T23:32:17.116Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.801955778Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178822023-09-22T23:32:17.119Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.802057551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178832023-09-22T23:32:17.122Z{"msg":"97c01c1d-4ab1-4912-a462-acf8f364d48d WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.802128969Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178842023-09-22T23:32:17.125Z{"msg":"[0] 97c01c1d-4ab1-4912-a462-acf8f364d48d (4b61aad5-6735-4028-b97c-dbf6ca91d9bf) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.804475592Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178852023-09-22T23:32:17.128Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.804576963Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178862023-09-22T23:32:17.131Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:16.80463428Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178872023-09-22T23:32:17.134Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.804675542Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178882023-09-22T23:32:17.137Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.804714364Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178892023-09-22T23:32:17.140Z{"msg":"97c01c1d-4ab1-4912-a462-acf8f364d48d WaitQuorum WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.804753512Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178902023-09-22T23:32:17.143Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.804792319Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178912023-09-22T23:32:17.146Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.80482976Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178922023-09-22T23:32:17.149Z{"msg":"[1] 97c01c1d-4ab1-4912-a462-acf8f364d48d (4b61aad5-6735-4028-b97c-dbf6ca91d9bf) WaitQuorum WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.804870753Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178932023-09-22T23:32:17.152Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.804909521Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178942023-09-22T23:32:17.155Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:16.804947162Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178952023-09-22T23:32:17.159Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.804984521Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178962023-09-22T23:32:17.162Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.805004359Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178972023-09-22T23:32:17.165Z{"msg":"97c01c1d-4ab1-4912-a462-acf8f364d48d WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.805022807Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178982023-09-22T23:32:17.168Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.805040447Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
178992023-09-22T23:32:17.170Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:16.805057797Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179002023-09-22T23:32:17.801Z{"msg":"[2] 97c01c1d-4ab1-4912-a462-acf8f364d48d looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.800822617Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721,"looper":"2"}
179012023-09-22T23:32:17.805Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.80090627Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179022023-09-22T23:32:17.807Z{"msg":"[2] 97c01c1d-4ab1-4912-a462-acf8f364d48d (4b61aad5-6735-4028-b97c-dbf6ca91d9bf) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.802117206Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179032023-09-22T23:32:17.810Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.80217373Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179042023-09-22T23:32:17.813Z{"msg":"[2] client is_active_req TRUE, promote! session 4b61aad5-6735-4028-b97c-dbf6ca91d9bf","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.802197793Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179052023-09-22T23:32:17.816Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.802845325Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179062023-09-22T23:32:17.819Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.80290855Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179072023-09-22T23:32:17.822Z{"msg":"97c01c1d-4ab1-4912-a462-acf8f364d48d WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.802951489Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179082023-09-22T23:32:17.825Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.805128309Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179092023-09-22T23:32:17.828Z{"msg":"[2] 97c01c1d-4ab1-4912-a462-acf8f364d48d (4b61aad5-6735-4028-b97c-dbf6ca91d9bf) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.805348011Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179102023-09-22T23:32:17.830Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.805403035Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179112023-09-22T23:32:17.834Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:17.805425829Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179122023-09-22T23:32:17.837Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.805463519Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179132023-09-22T23:32:17.840Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.805517845Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179142023-09-22T23:32:17.842Z{"msg":"97c01c1d-4ab1-4912-a462-acf8f364d48d WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.8055616Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179152023-09-22T23:32:17.845Z{"msg":"[0]R flush_numbers[0..12]: [183, 202, 197, 205, 203, 205, 204, 202, 204, 194, 200, 205]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.80561438Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179162023-09-22T23:32:17.847Z{"msg":"[0]R generation[0..12]: [121, 132, 130, 135, 133, 135, 134, 132, 134, 128, 131, 135]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.805657014Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179172023-09-22T23:32:17.850Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.805698081Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179182023-09-22T23:32:17.853Z{"msg":"[1]R flush_numbers[0..12]: [183, 202, 197, 205, 203, 205, 204, 202, 204, 194, 200, 205]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.805738198Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179192023-09-22T23:32:17.856Z{"msg":"[1]R generation[0..12]: [121, 132, 130, 135, 133, 135, 134, 132, 134, 128, 131, 135]"The guest has finished waiting for activation with:136
179202023-09-22T23:32:17.859ZWait for a query_work_queue command to finish before sending IO
179212023-09-22T23:32:17.861Z,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.805949321Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179222023-09-22T23:32:17.864Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.806036945Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179232023-09-22T23:32:17.867Z{"msg":"[2]R flush_numbers[0..12]: [183, 202, 197, 205, 203, 205, 204, 202, 204, 194, 200, 205]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.806091893Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179242023-09-22T23:32:17.870Z{"msg":"[2]R generation[0..12]: [121, 132, 130, 135, 133, 135, 134, 132, 134, 128, 131, 135]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.806133504Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179252023-09-22T23:32:17.873Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.806173835Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179262023-09-22T23:32:17.876Z{"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.806213331Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179272023-09-22T23:32:17.879Z{"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.806256021Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179282023-09-22T23:32:17.882Z{"msg":"Next flush: 206","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.806293763Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179292023-09-22T23:32:17.885Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.806331762Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179302023-09-22T23:32:17.887Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.806369666Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179312023-09-22T23:32:17.890Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.806412635Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179322023-09-22T23:32:17.893Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.806464129Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179332023-09-22T23:32:17.896Z{"msg":"97c01c1d-4ab1-4912-a462-acf8f364d48d is now active with session: 4b61aad5-6735-4028-b97c-dbf6ca91d9bf","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.806504862Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179342023-09-22T23:32:17.899Z{"msg":"97c01c1d-4ab1-4912-a462-acf8f364d48d Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.806544524Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179352023-09-22T23:32:17.902Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.806583153Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179362023-09-22T23:32:17.905Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.806621134Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179372023-09-22T23:32:17.911Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.806665976Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179382023-09-22T23:32:17.914Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.806704601Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179392023-09-22T23:32:17.917Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.806742361Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179402023-09-22T23:32:17.920Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.806781616Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179412023-09-22T23:32:17.923Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.806819924Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179422023-09-22T23:32:17.926Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
179432023-09-22T23:32:17.929Ztime":"2023-09-22T23:32:17.806858635Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179442023-09-22T23:32:17.932Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.806970404Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179452023-09-22T23:32:17.934Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.807015582Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179462023-09-22T23:32:17.937Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.807056575Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1721}
179472023-09-22T23:32:17.939ZLoading write count information from file "/tmp/test_repair_verify.data"
179482023-09-22T23:32:17.942ZRun Repair workload
179492023-09-22T23:32:17.945Z01/30 Write block 494 len 32768 data: 13 16 16 16 9 10 10 9
179502023-09-22T23:32:17.948Z02/30 Flush
179512023-09-22T23:32:17.950Z03/30 Write block 275 len 12288 data: 16 19 15
179522023-09-22T23:32:17.953Z04/30 Write block 201 len 36864 data: 10 12 14 13 19 17 16 19 16
179532023-09-22T23:32:17.956Z05/30 Write block 273 len 40960 data: 16 15 17 20 16 18 18 17 15 15
179542023-09-22T23:32:17.960Z06/30 Read block 158 len 28672
179552023-09-22T23:32:17.967Z07/30 Write block 513 len 12288 data: 15 17 15
179562023-09-22T23:32:17.970Z08/30 Write block 351 len 12288 data: 11 11 11
179572023-09-22T23:32:17.973Z09/30 Read block 52 len 4096
179582023-09-22T23:32:17.977Z10/30 Read block 28 len 40960
179592023-09-22T23:32:17.980Z11/30 Read block 243 len 12288
179602023-09-22T23:32:17.983Z12/30 Read block 514 len 28672
179612023-09-22T23:32:17.986Z13/30 Read block 484 len 24576
179622023-09-22T23:32:17.989Z14/30 Read block 517 len 24576
179632023-09-22T23:32:17.992Z15/30 Flush
179642023-09-22T23:32:17.994Z16/30 Write block 545 len 40960 data: 9 13 9 12 15 14 14 14 13 8
179652023-09-22T23:32:17.997Z17/30 Read block 241 len 12288
179662023-09-22T23:32:18.000Z18/30 Read block 4 len 4096
179672023-09-22T23:32:18.003Z19/30 Flush
179682023-09-22T23:32:18.006Z20/30 Write block 146 len 16384 data: 7 6 10 11
179692023-09-22T23:32:18.009Z21/30 Read block 176 len 12288
179702023-09-22T23:32:18.012Z22/30 Write block 438 len 32768 data: 13 11 9 12 12 14 15 14
179712023-09-22T23:32:18.015Z23/30 Read block 328 len 16384
179722023-09-22T23:32:18.019Z24/30 Read block 11 len 4096
179732023-09-22T23:32:18.021Z25/30 Write block 334 len 28672 data: 13 14 13 10 10 9 8
179742023-09-22T23:32:18.024Z26/30 Write block 329 len 36864 data: 10 7 10 10 10 14 15 14 11
179752023-09-22T23:32:18.027Z27/30 Write block 291 len 40960 data: 13 18 15 14 15 14 10 10 10 8
179762023-09-22T23:32:18.030Z28/30 Read block 292 len 40960
179772023-09-22T23:32:18.034Z29/30 Write block 489 len 12288 data: 20 17 18
179782023-09-22T23:32:18.037Z30/30 Read block 99 len 32768
179792023-09-22T23:32:18.040Z----------------------------------------------------------------
179802023-09-22T23:32:18.043Z Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:28
179812023-09-22T23:32:18.046ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
179822023-09-22T23:32:18.049Z 3 Acked 1002 Write 3 Done Done Done false
179832023-09-22T23:32:18.052Z 4 Acked 1003 Write 9 Done Done Sent false
179842023-09-22T23:32:18.055Z 5 Acked 1004 Write 10 Done Done Sent false
179852023-09-22T23:32:18.057Z 6 Acked 1005 Read 7 Done Done Sent false
179862023-09-22T23:32:18.060Z 7 Acked 1006 Write 3 Done Done Sent false
179872023-09-22T23:32:18.063Z 8 Acked 1007 Write 3 Done Done Sent false
179882023-09-22T23:32:18.066Z 9 Acked 1008 Read 1 Done Done Sent false
179892023-09-22T23:32:18.069Z 10 Acked 1009 Read 10 Done Done Sent false
179902023-09-22T23:32:18.072Z 11 Acked 1010 Read 3 Done Done Sent false
179912023-09-22T23:32:18.075Z 12 Acked 1011 Read 7 Done Done Sent false
179922023-09-22T23:32:18.078Z 13 Acked 1012 Read 6 Done Done Sent false
179932023-09-22T23:32:18.081Z 14 Acked 1013 Read 6 Done Done Sent false
179942023-09-22T23:32:18.084Z 15 Acked 1014 Flush 0 Done Done Sent false
179952023-09-22T23:32:18.087Z 16 Acked 1015 Write 10 Done Done Sent false
179962023-09-22T23:32:18.090Z 17 Acked 1016 Read 3 Done Done Sent false
179972023-09-22T23:32:18.092Z 18 Acked 1017 Read 1 Done Done Sent false
179982023-09-22T23:32:18.095Z 19 Acked 1018 Flush 0 Done Done Sent false
179992023-09-22T23:32:18.098Z 20 Acked 1019 Write 4 Done Done Sent false
180002023-09-22T23:32:18.101Z 21 Acked 1020 Read 3 Done Done Sent false
180012023-09-22T23:32:18.104Z 22 Acked 1021 Write 8 Done Done Sent false
180022023-09-22T23:32:18.107Z 23 Acked 1022 Read 4 Done Done Sent false
180032023-09-22T23:32:18.110Z 24 Acked 1023 Read 1 Done Done Sent false
180042023-09-22T23:32:18.113Z 25 Acked 1024 Write 7 Done Done Sent false
180052023-09-22T23:32:18.118Z 26 Acked 1025 Write 9 Done Done Sent false
180062023-09-22T23:32:18.120Z 27 Acked 1026 Write 10 Done Done Sent false
180072023-09-22T23:32:18.123Z 28 Acked 1027 Read 10 Done Done Sent false
180082023-09-22T23:32:18.126Z 29 Acked 1028 Write 3 Done Done Sent false
180092023-09-22T23:32:18.129Z 30 Acked 1029 Read 8 Sent Done Sent false
180102023-09-22T23:32:18.132Z STATES DS:0 DS:1 DS:2 TOTAL
180112023-09-22T23:32:18.135Z New 0 0 0 0
180122023-09-22T23:32:18.138Z Sent 1 0 27 28
180132023-09-22T23:32:18.141Z Done 27 28 1 56
180142023-09-22T23:32:18.144Z Skipped 0 0 0 0
180152023-09-22T23:32:18.147Z Error 0 0 0 0
180162023-09-22T23:32:18.149ZLast Flush: 1018 1018 1001
180172023-09-22T23:32:18.152ZDownstairs last five completed: 1001 1000
180182023-09-22T23:32:18.155ZUpstairs last five completed: 30 29 28 27 26 25
180192023-09-22T23:32:18.158ZWrote out file "/tmp/test_repair_verify.data"
180202023-09-22T23:32:18.161Z
180212023-09-22T23:32:18.163ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
180222023-09-22T23:32:18.166Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.95579691Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1722}
180232023-09-22T23:32:18.169Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.956108549Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1722}
180242023-09-22T23:32:18.171Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.956132139Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1722}
180252023-09-22T23:32:18.175Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.98734515Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1722}
180262023-09-22T23:32:18.177Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.987400825Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1722}
180272023-09-22T23:32:18.181Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:17.987440211Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1722}
180282023-09-22T23:32:18.183Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:18.009827028Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1722}
180292023-09-22T23:32:18.186Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:18.00989254Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1722}
180302023-09-22T23:32:18.189Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:18.009933949Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1722}
180312023-09-22T23:32:18.192ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
180322023-09-22T23:32:18.195Z 0 000-019 121 121 121 183 183 183 F F F
180332023-09-22T23:32:18.198Z 1 020-039 132 132 132 202 202 202 F F F
180342023-09-22T23:32:18.200Z 2 040-059 130 130 130 197 197 197 F F F
180352023-09-22T23:32:18.203Z 3 060-079 135 135 135 205 205 205 F F F
180362023-09-22T23:32:18.207Z 4 080-099 133 133 133 203 203 203 F F F
180372023-09-22T23:32:18.209Z 5 100-119 135 135 135 205 205 205 F F F
180382023-09-22T23:32:18.212Z 6 120-139 134 134 134 204 204 204 F F F
180392023-09-22T23:32:18.215Z 7 140-159 132 132 132 202 202 202 T T F <---
180402023-09-22T23:32:18.217Z 8 160-179 134 134 134 204 204 204 F F F
180412023-09-22T23:32:18.220Z 9 180-199 128 128 128 194 194 194 F F F
180422023-09-22T23:32:18.223Z 10 200-219 136 136 131 207 207 200 F F F <---
180432023-09-22T23:32:18.226Z 11 220-239 135 135 135 205 205 205 F F F
180442023-09-22T23:32:18.229Z 12 240-259 135 135 135 205 205 205 F F F
180452023-09-22T23:32:18.231ZError: Difference in extent metadata found!
180462023-09-22T23:32:18.255Z 13 260-279 136 136 129 207 207 196 F F T <---
180472023-09-22T23:32:18.259Z 14 280-299 136 136 133 207 207 203 T T F <---
180482023-09-22T23:32:18.266Z 15 300-319 133 133 133 203 203 203 T T F <---
180492023-09-22T23:32:18.269Z 16 320-339 135 135 135 205 205 205 T T F <---
180502023-09-22T23:32:18.272Z 17 340-359 136 136 135 207 207 205 T T F <---
180512023-09-22T23:32:18.275Z 18 360-379 134 134 134 204 204 204 F F F
180522023-09-22T23:32:18.277Z 19 380-399 135 135 135 205 205 205 F F F
180532023-09-22T23:32:18.280Z 20 400-419 135 135 135 205 205 205 F F F
180542023-09-22T23:32:18.283Z 21 420-439 132 132 132 202 202 202 T T F <---
180552023-09-22T23:32:18.285Z 22 440-459 135 135 135 205 205 205 T T F <---
180562023-09-22T23:32:18.288Z 23 460-479 134 134 134 204 204 204 F F F
180572023-09-22T23:32:18.291Z 24 480-499 136 136 136 206 206 206 T T F <---
180582023-09-22T23:32:18.294Z 25 500-519 136 136 136 207 207 206 F F F <---
180592023-09-22T23:32:18.297Z 26 520-539 125 125 125 190 190 190 F F F
180602023-09-22T23:32:18.299Z 27 540-559 136 136 132 208 208 202 F F F <---
180612023-09-22T23:32:18.302Z 28 560-579 133 133 133 203 203 203 F F F
180622023-09-22T23:32:18.305Z 29 580-599 130 130 130 197 197 197 F F F
180632023-09-22T23:32:18.308ZMax gen: 136, Max flush: 208
180642023-09-22T23:32:18.310ZA Difference in extent metadata is expected here
180652023-09-22T23:32:18.313ZOn loop 68
180662023-09-22T23:32:18.316Z
180672023-09-22T23:32:18.318ZVerifying data now
180682023-09-22T23:32:24.417ZNo change after verify
180692023-09-22T23:32:24.424ZLoop: 68 Downstairs dump after verify (and repair):
180702023-09-22T23:32:24.429Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.401855474Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1727}
180712023-09-22T23:32:24.436Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.402244008Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1727}
180722023-09-22T23:32:24.447Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.402286935Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1727}
180732023-09-22T23:32:24.453Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.421405589Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1727}
180742023-09-22T23:32:24.458Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.421457268Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1727}
180752023-09-22T23:32:24.464Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.421495579Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1727}
180762023-09-22T23:32:24.470Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.432730835Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1727}
180772023-09-22T23:32:24.474Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.432784307Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1727}
180782023-09-22T23:32:24.478Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.432837355Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1727}
180792023-09-22T23:32:24.482ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
180802023-09-22T23:32:24.487Z 0 000-019 121 121 121 183 183 183 F F F
180812023-09-22T23:32:24.490Z 1 020-039 132 132 132 202 202 202 F F F
180822023-09-22T23:32:24.494Z 2 040-059 130 130 130 197 197 197 F F F
180832023-09-22T23:32:24.498Z 3 060-079 135 135 135 205 205 205 F F F
180842023-09-22T23:32:24.501Z 4 080-099 133 133 133 203 203 203 F F F
180852023-09-22T23:32:24.504Z 5 100-119 135 135 135 205 205 205 F F F
180862023-09-22T23:32:24.507Z 6 120-139 134 134 134 204 204 204 F F F
180872023-09-22T23:32:24.509Z 7 140-159 137 137 137 209 209 209 F F F
180882023-09-22T23:32:24.512Z 8 160-179 134 134 134 204 204 204 F F F
180892023-09-22T23:32:24.515Z 9 180-199 128 128 128 194 194 194 F F F
180902023-09-22T23:32:24.517Z 10 200-219 136 136 136 207 207 207 F F F
180912023-09-22T23:32:24.520Z 11 220-239 135 135 135 205 205 205 F F F
180922023-09-22T23:32:24.523Z 12 240-259 135 135 135 205 205 205 F F F
180932023-09-22T23:32:24.526Z 13 260-279 136 136 136 207 207 207 F F F
180942023-09-22T23:32:24.529Z 14 280-299 137 137 137 209 209 209 F F F
180952023-09-22T23:32:24.531Z 15 300-319 137 137 137 209 209 209 F F F
180962023-09-22T23:32:24.534Z 16 320-339 137 137 137 209 209 209 F F F
180972023-09-22T23:32:24.537Z 17 340-359 137 137 137 209 209 209 F F F
180982023-09-22T23:32:24.539Z 18 360-379 134 134 134 204 204 204 F F F
180992023-09-22T23:32:24.542Z 19 380-399 135 135 135 205 205 205 F F F
181002023-09-22T23:32:24.544Z 20 400-419 135 135 135 205 205 205 F F F
181012023-09-22T23:32:24.547Z 21 420-439 137 137 137 209 209 209 F F F
181022023-09-22T23:32:24.550Z 22 440-459 137 137 137 209 209 209 F F F
181032023-09-22T23:32:24.553Z 23 460-479 134 134 134 204 204 204 F F F
181042023-09-22T23:32:24.555Z 24 480-499 137 137 137 209 209 209 F F F
181052023-09-22T23:32:24.558Z 25 500-519 136 136 136 207 207 207 F F F
181062023-09-22T23:32:24.561Z 26 520-539 125 125 125 190 190 190 F F F
181072023-09-22T23:32:24.563Z 27 540-559 136 136 136 208 208 208 F F F
181082023-09-22T23:32:24.566Z 28 560-579 133 133 133 203 203 203 F F F
181092023-09-22T23:32:24.569Z 29 580-599 130 130 130 197 197 197 F F F
181102023-09-22T23:32:24.571ZMax gen: 137, Max flush: 209
181112023-09-22T23:32:24.574Z
181122023-09-22T23:32:24.577ZBegin loop 69
181132023-09-22T23:32:24.579ZDownstairs to restart: 1
181142023-09-22T23:32:24.582Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.483354067Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181152023-09-22T23:32:24.585Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.483733501Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181162023-09-22T23:32:24.588Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.483811659Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181172023-09-22T23:32:24.591Z{"msg":"Crucible stats registered with UUID: 23e53e81-8304-48bf-bf88-a44af676d9d9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.483932932Z","hostname":"Crucible runtime is spawned
181182023-09-22T23:32:24.593Zip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181192023-09-22T23:32:24.596Z{"msg":"Crucible 23e53e81-8304-48bf-bf88-a44af676d9d9 has session id: 7bdc74a2-195c-4edc-82bc-cce441a4316d","v":0,"name":"crucible","level":30The guest has requested activation with gen:138
181202023-09-22T23:32:24.600Z,"time":"2023-09-22T23:32:24.484073856Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181212023-09-22T23:32:24.603Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.484157257Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729,"task":"up_listen"}
181222023-09-22T23:32:24.606Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.484193815Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181232023-09-22T23:32:24.609Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.484247891Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181242023-09-22T23:32:24.611Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.484311635Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729,"looper":"1"}
181252023-09-22T23:32:24.614Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.484508277Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729,"looper":"2"}
181262023-09-22T23:32:24.617Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.484711223Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729,"looper":"0"}
181272023-09-22T23:32:24.620Z{"msg":"[2] 23e53e81-8304-48bf-bf88-a44af676d9d9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.484923368Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729,"looper":"2"}
181282023-09-22T23:32:24.623Z{"msg":"[0] 23e53e81-8304-48bf-bf88-a44af676d9d9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.485017495Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729,"looper":"0"}
181292023-09-22T23:32:24.625Z{"msg":"23e53e81-8304-48bf-bf88-a44af676d9d9 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.485071515Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181302023-09-22T23:32:24.628Z{"msg":"Set desired generation to :138","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.48513075Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181312023-09-22T23:32:24.630Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.485154786Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181322023-09-22T23:32:24.633Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.485594826Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181332023-09-22T23:32:24.636Z{"msg":"[2] 23e53e81-8304-48bf-bf88-a44af676d9d9 (a88ef3e5-bdb0-455c-8a7d-e583d45c2c23) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.485736822Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181342023-09-22T23:32:24.639Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.485820763Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181352023-09-22T23:32:24.642Z{"msg":"[2] client is_active_req TRUE, promote! session a88ef3e5-bdb0-455c-8a7d-e583d45c2c23","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.485905047Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181362023-09-22T23:32:24.645Z{"msg":"[0] 23e53e81-8304-48bf-bf88-a44af676d9d9 (a88ef3e5-bdb0-455c-8a7d-e583d45c2c23) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.485982621Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181372023-09-22T23:32:24.647Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.486056495Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181382023-09-22T23:32:24.650Z{"msg":"[0] client is_active_req TRUE, promote! session a88ef3e5-bdb0-455c-8a7d-e583d45c2c23","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.486119824Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181392023-09-22T23:32:24.652Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.486551689Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181402023-09-22T23:32:24.655Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.486618928Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181412023-09-22T23:32:24.658Z{"msg":"23e53e81-8304-48bf-bf88-a44af676d9d9 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.486697911Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181422023-09-22T23:32:24.660Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.48682323Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181432023-09-22T23:32:24.663Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.486902101Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181442023-09-22T23:32:24.666Z{"msg":"23e53e81-8304-48bf-bf88-a44af676d9d9 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.486986419Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181452023-09-22T23:32:24.670Z{"msg":"[2] 23e53e81-8304-48bf-bf88-a44af676d9d9 (a88ef3e5-bdb0-455c-8a7d-e583d45c2c23) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.488956065Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181462023-09-22T23:32:24.676Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.489057061Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181472023-09-22T23:32:24.684Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:24.489089525Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181482023-09-22T23:32:24.691Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.48920732Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181492023-09-22T23:32:24.695Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.48931653Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181502023-09-22T23:32:24.703Z{"msg":"23e53e81-8304-48bf-bf88-a44af676d9d9 WaitActive New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.489342662Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181512023-09-22T23:32:24.709Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.489361139Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181522023-09-22T23:32:24.716Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.489378276Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181532023-09-22T23:32:24.724Z{"msg":"[0] 23e53e81-8304-48bf-bf88-a44af676d9d9 (a88ef3e5-bdb0-455c-8a7d-e583d45c2c23) WaitActive New WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.489560754Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181542023-09-22T23:32:24.730Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.48958652Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181552023-09-22T23:32:24.738Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:24.48960499Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181562023-09-22T23:32:24.746Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.48962212Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181572023-09-22T23:32:24.755Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.489640274Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181582023-09-22T23:32:24.763Z{"msg":"23e53e81-8304-48bf-bf88-a44af676d9d9 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.489658755Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181592023-09-22T23:32:24.771Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.489676981Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181602023-09-22T23:32:24.778Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:24.489694461Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181612023-09-22T23:32:25.877Z{"msg":"[1] 23e53e81-8304-48bf-bf88-a44af676d9d9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.485547231Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729,"looper":"1"}
181622023-09-22T23:32:25.884Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.485654886Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181632023-09-22T23:32:25.901Z{"msg":"[1] 23e53e81-8304-48bf-bf88-a44af676d9d9 (a88ef3e5-bdb0-455c-8a7d-e583d45c2c23) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.486640348Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181642023-09-22T23:32:25.915Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.486725695Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181652023-09-22T23:32:25.930Z{"msg":"[1] client is_active_req TRUE, promote! session a88ef3e5-bdb0-455c-8a7d-e583d45c2c23","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.486818057Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181662023-09-22T23:32:25.940Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.487346106Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181672023-09-22T23:32:25.951Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.48742644Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181682023-09-22T23:32:25.967Z{"msg":"23e53e81-8304-48bf-bf88-a44af676d9d9 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.487489363Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181692023-09-22T23:32:25.977Z{"msg":"[1] 23e53e81-8304-48bf-bf88-a44af676d9d9 (a88ef3e5-bdb0-455c-8a7d-e583d45c2c23) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.489806092Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181702023-09-22T23:32:25.994Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.489883827Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181712023-09-22T23:32:26.414Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:25.48995659Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181722023-09-22T23:32:26.426Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.490022241Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181732023-09-22T23:32:26.432Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.490091159Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181742023-09-22T23:32:26.438Z{"msg":"23e53e81-8304-48bf-bf88-a44af676d9d9 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.490158504Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181752023-09-22T23:32:26.446Z{"msg":"[0]R flush_numbers[0..12]: [183, 202, 197, 205, 203, 205, 204, 209, 204, 194, 207, 205]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.490223439Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181762023-09-22T23:32:26.451Z{"msg":"[0]R generation[0..12]: [121, 132, 130, 135, 133, 135, 134, 137, 134, 128, 136, 135]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.490291786Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181772023-09-22T23:32:26.506Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","The guest has finished waiting for activation with:138
181782023-09-22T23:32:26.511ZWait for a query_work_queue command to finish before sending IO
181792023-09-22T23:32:26.518Zlevel":30,"time":"2023-09-22T23:32:25.490501104Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181802023-09-22T23:32:26.526Z{"msg":"[1]R flush_numbers[0..12]: [183, 202, 197, 205, 203, 205, 204, 209, 204, 194, 207, 205]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.490610385Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181812023-09-22T23:32:26.534Z{"msg":"[1]R generation[0..12]: [121, 132, 130, 135, 133, 135, 134, 137, 134, 128, 136, 135]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.49065244Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181822023-09-22T23:32:26.543Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.490720424Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181832023-09-22T23:32:26.551Z{"msg":"[2]R flush_numbers[0..12]: [183, 202, 197, 205, 203, 205, 204, 209, 204, 194, 207, 205]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.490768386Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181842023-09-22T23:32:26.557Z{"msg":"[2]R generation[0..12]: [121, 132, 130, 135, 133, 135, 134, 137, 134, 128, 136, 135]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.490828551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181852023-09-22T23:32:26.560Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.490882943Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181862023-09-22T23:32:26.563Z{"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.490924183Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181872023-09-22T23:32:26.567Z{"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.490973059Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181882023-09-22T23:32:26.571Z{"msg":"Next flush: 210","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.491010723Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181892023-09-22T23:32:26.576Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.491059187Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181902023-09-22T23:32:26.580Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.49111994Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181912023-09-22T23:32:26.584Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.491172636Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181922023-09-22T23:32:26.591Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.49121593Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181932023-09-22T23:32:26.597Z{"msg":"23e53e81-8304-48bf-bf88-a44af676d9d9 is now active with session: a88ef3e5-bdb0-455c-8a7d-e583d45c2c23","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.491256392Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181942023-09-22T23:32:26.605Z{"msg":"23e53e81-8304-48bf-bf88-a44af676d9d9 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.491295138Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181952023-09-22T23:32:26.617Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.491333508Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181962023-09-22T23:32:26.625Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.491375513Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181972023-09-22T23:32:26.633Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.49141392Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181982023-09-22T23:32:26.638Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.491452018Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
181992023-09-22T23:32:26.647Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.491490564Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
182002023-09-22T23:32:26.655Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.491528023Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
182012023-09-22T23:32:26.665Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.491566237Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
182022023-09-22T23:32:26.675Zip-10-150-1-177.us-west-2.compute.internal","pid":1729}
182032023-09-22T23:32:26.682Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.491677335Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
182042023-09-22T23:32:26.689Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.491722114Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
182052023-09-22T23:32:26.696Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.491760157Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
182062023-09-22T23:32:26.706Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.491808186Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1729}
182072023-09-22T23:32:26.716ZLoading write count information from file "/tmp/test_repair_verify.data"
182082023-09-22T23:32:26.725ZRun Repair workload
182092023-09-22T23:32:26.731Z01/30 Write block 175 len 20480 data: 17 19 17 16 12
182102023-09-22T23:32:26.739Z02/30 Write block 578 len 28672 data: 10 8 9 9 9 8 8
182112023-09-22T23:32:26.748Z03/30 Write block 430 len 28672 data: 15 13 15 13 11 14 13
182122023-09-22T23:32:26.758Z04/30 Read block 494 len 36864
182132023-09-22T23:32:26.770Z05/30 Read block 516 len 4096
182142023-09-22T23:32:26.788Z06/30 Read block 195 len 36864
182152023-09-22T23:32:26.808Z07/30 Write block 438 len 24576 data: 14 12 10 13 13 15
182162023-09-22T23:32:26.819Z08/30 Read block 286 len 36864
182172023-09-22T23:32:26.835Z09/30 Write block 320 len 8192 data: 11 11
182182023-09-22T23:32:26.845Z10/30 Read block 110 len 16384
182192023-09-22T23:32:26.856Z11/30 Flush
182202023-09-22T23:32:26.870Z12/30 Write block 331 len 24576 data: 11 11 11 15 16 15
182212023-09-22T23:32:26.879Z13/30 Read block 30 len 24576
182222023-09-22T23:32:26.889Z14/30 Write block 93 len 4096 data: 11
182232023-09-22T23:32:26.952Z15/30 Read block 127 len 8192
182242023-09-22T23:32:26.967Z16/30 Write block 533 len 4096 data: 13
182252023-09-22T23:32:26.982Z17/30 Write block 204 len 28672 data: 14 20 18 17 20 17 17
182262023-09-22T23:32:27.000ZError: Difference in extent metadata found!
182272023-09-22T23:32:27.013Z18/30 Write block 202 len 8192 data: 13 15
182282023-09-22T23:32:27.031Z19/30 Read block 296 len 36864
182292023-09-22T23:32:27.043Z20/30 Read block 587 len 20480
182302023-09-22T23:32:27.052Z21/30 Read block 278 len 32768
182312023-09-22T23:32:27.062Z22/30 Read block 507 len 8192
182322023-09-22T23:32:27.070Z23/30 Read block 427 len 28672
182332023-09-22T23:32:27.079Z24/30 Read block 63 len 12288
182342023-09-22T23:32:27.091Z25/30 Write block 258 len 24576 data: 10 9 9 12 13 14
182352023-09-22T23:32:27.107Z26/30 Read block 545 len 28672
182362023-09-22T23:32:27.125Z27/30 Read block 106 len 4096
182372023-09-22T23:32:27.141Z28/30 Read block 38 len 36864
182382023-09-22T23:32:27.150Z29/30 Write block 529 len 28672 data: 10 8 10 11 14 10 12
182392023-09-22T23:32:27.161Z30/30 Read block 2 len 4096
182402023-09-22T23:32:27.171Z----------------------------------------------------------------
182412023-09-22T23:32:27.181Z Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:30
182422023-09-22T23:32:27.193ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
182432023-09-22T23:32:27.202Z 1 Acked 1000 Write 5 Done Done Done false
182442023-09-22T23:32:27.213Z 2 Acked 1001 Write 7 Done Done Done false
182452023-09-22T23:32:27.222Z 3 Acked 1002 Write 7 Done Done Done false
182462023-09-22T23:32:27.229Z 4 Acked 1003 Read 9 Done Done Done false
182472023-09-22T23:32:27.242Z 5 Acked 1004 Read 1 Done Sent Done false
182482023-09-22T23:32:27.253Z 6 Acked 1005 Read 9 Done Sent Done false
182492023-09-22T23:32:27.264Z 7 Acked 1006 Write 6 Done Sent Done false
182502023-09-22T23:32:27.285Z 8 Acked 1007 Read 9 Done Sent Done false
182512023-09-22T23:32:27.295Z 9 Acked 1008 Write 2 Done Sent Done false
182522023-09-22T23:32:27.308Z 10 Acked 1009 Read 4 Done Sent Done false
182532023-09-22T23:32:27.321Z 11 Acked 1010 Flush 0 Done Sent Done false
182542023-09-22T23:32:27.742Z 12 Acked 1011 Write 6 Done Sent Done false
182552023-09-22T23:32:27.747Z 13 Acked 1012 Read 6 Done Sent Done false
182562023-09-22T23:32:28.163Z 14 Acked 1013 Write 1 Done Sent Done false
182572023-09-22T23:32:28.170Z 15 Acked 1014 Read 2 Done Sent Done false
182582023-09-22T23:32:28.175Z 16 Acked 1015 Write 1 Done Sent Done false
182592023-09-22T23:32:28.179Z 17 Acked 1016 Write 7 Done Sent Done false
182602023-09-22T23:32:28.187Z 18 Acked 1017 Write 2 Done Sent Done false
182612023-09-22T23:32:28.196Z 19 Acked 1018 Read 9 Done Sent Done false
182622023-09-22T23:32:28.206Z 20 Acked 1019 Read 5 Done Sent Done false
182632023-09-22T23:32:28.216Z 21 Acked 1020 Read 8 Done Sent Done false
182642023-09-22T23:32:28.234Z 22 Acked 1021 Read 2 Done Sent Done false
182652023-09-22T23:32:28.244Z 23 Acked 1022 Read 7 Done Sent Done false
182662023-09-22T23:32:28.252Z 24 Acked 1023 Read 3 Done Sent Done false
182672023-09-22T23:32:28.258Z 25 Acked 1024 Write 6 Done Sent Done false
182682023-09-22T23:32:28.264Z 26 Acked 1025 Read 7 Done Sent Done false
182692023-09-22T23:32:28.287Z 27 Acked 1026 Read 1 Done Sent Done false
182702023-09-22T23:32:28.290Z 28 Acked 1027 Read 9 Done Sent Done false
182712023-09-22T23:32:28.294Z 29 Acked 1028 Write 7 Done Sent Done false
182722023-09-22T23:32:28.298Z 30 Acked 1029 Read 1 Done Sent Done false
182732023-09-22T23:32:28.301Z STATES DS:0 DS:1 DS:2 TOTAL
182742023-09-22T23:32:28.304Z New 0 0 0 0
182752023-09-22T23:32:28.308Z Sent 0 26 0 26
182762023-09-22T23:32:28.311Z Done 30 4 30 64
182772023-09-22T23:32:28.314Z Skipped 0 0 0 0
182782023-09-22T23:32:28.317Z Error 0 0 0 0
182792023-09-22T23:32:28.320ZLast Flush: 1010 0 1010
182802023-09-22T23:32:28.324ZDownstairs last five completed:
182812023-09-22T23:32:28.332ZUpstairs last five completed: 30 29 28 27 26 25
182822023-09-22T23:32:28.345ZWrote out file "/tmp/test_repair_verify.data"
182832023-09-22T23:32:28.369Z
182842023-09-22T23:32:28.378ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
182852023-09-22T23:32:28.386Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.623874364Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1730}
182862023-09-22T23:32:28.392Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.624209484Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1730}
182872023-09-22T23:32:28.397Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.624232958Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1730}
182882023-09-22T23:32:28.400Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.652354363Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1730}
182892023-09-22T23:32:28.404Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.652393047Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1730}
182902023-09-22T23:32:28.409Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.652412927Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1730}
182912023-09-22T23:32:28.416Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.685170832Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1730}
182922023-09-22T23:32:28.425Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.685208932Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1730}
182932023-09-22T23:32:28.438Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:25.685228668Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1730}
182942023-09-22T23:32:28.464ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
182952023-09-22T23:32:28.495Z 0 000-019 121 121 121 183 183 183 F F F
182962023-09-22T23:32:28.515Z 1 020-039 132 132 132 202 202 202 F F F
182972023-09-22T23:32:28.532Z 2 040-059 130 130 130 197 197 197 F F F
182982023-09-22T23:32:28.572Z 3 060-079 135 135 135 205 205 205 F F F
182992023-09-22T23:32:28.578Z 4 080-099 133 133 133 203 203 203 T F T <---
183002023-09-22T23:32:28.581Z 5 100-119 135 135 135 205 205 205 F F F
183012023-09-22T23:32:28.584Z 6 120-139 134 134 134 204 204 204 F F F
183022023-09-22T23:32:28.587Z 7 140-159 137 137 137 209 209 209 F F F
183032023-09-22T23:32:28.591Z 8 160-179 138 134 138 210 204 210 F T F <---
183042023-09-22T23:32:28.594Z 9 180-199 128 128 128 194 194 194 F F F
183052023-09-22T23:32:28.597Z 10 200-219 136 136 136 207 207 207 T F T <---
183062023-09-22T23:32:28.600Z 11 220-239 135 135 135 205 205 205 F F F
183072023-09-22T23:32:28.604Z 12 240-259 135 135 135 205 205 205 T F T <---
183082023-09-22T23:32:28.607Z 13 260-279 136 136 136 207 207 207 T F T <---
183092023-09-22T23:32:28.610Z 14 280-299 137 137 137 209 209 209 F F F
183102023-09-22T23:32:28.613Z 15 300-319 137 137 137 209 209 209 F F F
183112023-09-22T23:32:28.616Z 16 320-339 138 137 138 210 209 210 T F T <---
183122023-09-22T23:32:28.619Z 17 340-359 137 137 137 209 209 209 F F F
183132023-09-22T23:32:28.621Z 18 360-379 134 134 134 204 204 204 F F F
183142023-09-22T23:32:28.624Z 19 380-399 135 135 135 205 205 205 F F F
183152023-09-22T23:32:28.627Z 20 400-419 135 135 135 205 205 205 F F F
183162023-09-22T23:32:28.630Z 21 420-439 138 137 138 210 209 210 F T F <---
183172023-09-22T23:32:28.633Z 22 440-459 138 137 138 210 209 210 F F F <---
183182023-09-22T23:32:28.636Z 23 460-479 134 134 134 204 204 204 F F F
183192023-09-22T23:32:28.639Z 24 480-499 137 137 137 209 209 209 F F F
183202023-09-22T23:32:28.642Z 25 500-519 136 136 136 207 207 207 F F F
183212023-09-22T23:32:29.061Z 26 520-539 125 125 125 190 190 190 T F T <---
183222023-09-22T23:32:29.070Z 27 540-559 136 136 136 208 208 208 F F F
183232023-09-22T23:32:29.079Z 28 560-579 138 133 138 210 203 210 F T F <---
183242023-09-22T23:32:29.090Z 29 580-599 138 130 138 210 197 210 F T F <---
183252023-09-22T23:32:29.101ZMax gen: 138, Max flush: 210
183262023-09-22T23:32:29.115ZA Difference in extent metadata is expected here
183272023-09-22T23:32:29.128ZOn loop 69
183282023-09-22T23:32:29.548Z
183292023-09-22T23:32:29.560ZVerifying data now
183302023-09-22T23:32:31.967ZNo change after verify
183312023-09-22T23:32:31.974ZLoop: 69 Downstairs dump after verify (and repair):
183322023-09-22T23:32:31.983Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:31.981495934Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1735}
183332023-09-22T23:32:31.988Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:31.981921356Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1735}
183342023-09-22T23:32:31.998Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:31.981946407Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1735}
183352023-09-22T23:32:32.005Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.000942335Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1735}
183362023-09-22T23:32:32.014Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.001002781Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1735}
183372023-09-22T23:32:32.019Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.001043433Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1735}
183382023-09-22T23:32:32.026Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.012097548Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1735}
183392023-09-22T23:32:32.031Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.012152705Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1735}
183402023-09-22T23:32:32.035Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.012190789Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1735}
183412023-09-22T23:32:32.041ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
183422023-09-22T23:32:32.045Z 0 000-019 121 121 121 183 183 183 F F F
183432023-09-22T23:32:32.049Z 1 020-039 132 132 132 202 202 202 F F F
183442023-09-22T23:32:32.052Z 2 040-059 130 130 130 197 197 197 F F F
183452023-09-22T23:32:32.059Z 3 060-079 135 135 135 205 205 205 F F F
183462023-09-22T23:32:32.062Z 4 080-099 139 139 139 211 211 211 F F F
183472023-09-22T23:32:32.068Z 5 100-119 135 135 135 205 205 205 F F F
183482023-09-22T23:32:32.077Z 6 120-139 134 134 134 204 204 204 F F F
183492023-09-22T23:32:32.081Z 7 140-159 137 137 137 209 209 209 F F F
183502023-09-22T23:32:32.087Z 8 160-179 138 138 138 210 210 210 F F F
183512023-09-22T23:32:32.093Z 9 180-199 128 128 128 194 194 194 F F F
183522023-09-22T23:32:32.101Z 10 200-219 139 139 139 211 211 211 F F F
183532023-09-22T23:32:32.108Z 11 220-239 135 135 135 205 205 205 F F F
183542023-09-22T23:32:32.115Z 12 240-259 139 139 139 211 211 211 F F F
183552023-09-22T23:32:32.529Z 13 260-279 139 139 139 211 211 211 F F F
183562023-09-22T23:32:32.532Z 14 280-299 137 137 137 209 209 209 F F F
183572023-09-22T23:32:32.535Z 15 300-319 137 137 137 209 209 209 F F F
183582023-09-22T23:32:32.540Z 16 320-339 139 139 139 211 211 211 F F F
183592023-09-22T23:32:32.545Z 17 340-359 137 137 137 209 209 209 F F F
183602023-09-22T23:32:32.557Z 18 360-379 134 134 134 204 204 204 F F F
183612023-09-22T23:32:32.570Z 19 380-399 135 135 135 205 205 205 F F F
183622023-09-22T23:32:32.586Z 20 400-419 135 135 135 205 205 205 F F F
183632023-09-22T23:32:32.591Z 21 420-439 138 138 138 210 210 210 F F F
183642023-09-22T23:32:32.596Z 22 440-459 138 138 138 210 210 210 F F F
183652023-09-22T23:32:32.601Z 23 460-479 134 134 134 204 204 204 F F F
183662023-09-22T23:32:32.606Z 24 480-499 137 137 137 209 209 209 F F F
183672023-09-22T23:32:32.610Z 25 500-519 136 136 136 207 207 207 F F F
183682023-09-22T23:32:32.617Z 26 520-539 139 139 139 211 211 211 F F F
183692023-09-22T23:32:32.624Z 27 540-559 136 136 136 208 208 208 F F F
183702023-09-22T23:32:32.627Z 28 560-579 138 138 138 210 210 210 F F F
183712023-09-22T23:32:32.630Z 29 580-599 138 138 138 210 210 210 F F F
183722023-09-22T23:32:32.634ZMax gen: 139, Max flush: 211
183732023-09-22T23:32:32.637Z
183742023-09-22T23:32:32.640ZBegin loop 70
183752023-09-22T23:32:32.644ZDownstairs to restart: 2
183762023-09-22T23:32:32.647Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.062388538Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
183772023-09-22T23:32:32.650Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.062757682Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
183782023-09-22T23:32:32.653Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.062834665Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
183792023-09-22T23:32:32.656Z{"msg":"Crucible stats registered with UUID: 9a30a600-1e9c-4c70-a202-c7991d534ac0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.062902509Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
183802023-09-22T23:32:32.659Z{"msg":"Crucible 9a30a600-1e9c-4c70-a202-c7991d534ac0 has session id: f3179ff1-a335-4873-8149-cc8cd431ddc0","v":0,"name":"crucible","level":30Crucible runtime is spawned
183812023-09-22T23:32:32.685Z,"time":"2023-09-22T23:32:32.062975136Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
183822023-09-22T23:32:32.688ZThe guest has requested activation with gen:140
183832023-09-22T23:32:32.691Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.063242511Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737,"task":"up_listen"}
183842023-09-22T23:32:32.694Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.063311483Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
183852023-09-22T23:32:32.696Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.063355441Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
183862023-09-22T23:32:32.699Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.063394874Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737,"looper":"0"}
183872023-09-22T23:32:32.702Z{"msg":"9a30a600-1e9c-4c70-a202-c7991d534ac0 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.063459704Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
183882023-09-22T23:32:32.705Z{"msg":"Set desired generation to :140","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.06352575Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
183892023-09-22T23:32:32.708Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.06367468Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737,"looper":"2"}
183902023-09-22T23:32:32.711Z{"msg":"[0] 9a30a600-1e9c-4c70-a202-c7991d534ac0 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.063744349Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737,"looper":"0"}
183912023-09-22T23:32:32.714Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.063787762Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737,"looper":"1"}
183922023-09-22T23:32:32.717Z{"msg":"[1] 9a30a600-1e9c-4c70-a202-c7991d534ac0 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.063961995Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737,"looper":"1"}
183932023-09-22T23:32:32.720Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.064037418Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
183942023-09-22T23:32:32.722Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.064264781Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
183952023-09-22T23:32:32.725Z{"msg":"[0] 9a30a600-1e9c-4c70-a202-c7991d534ac0 (38f823bf-6efb-41ed-8dda-a53bf86b0e2c) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.064896514Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
183962023-09-22T23:32:32.728Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.06495191Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
183972023-09-22T23:32:32.730Z{"msg":"[0] client is_active_req TRUE, promote! session 38f823bf-6efb-41ed-8dda-a53bf86b0e2c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.065029656Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
183982023-09-22T23:32:32.733Z{"msg":"[1] 9a30a600-1e9c-4c70-a202-c7991d534ac0 (38f823bf-6efb-41ed-8dda-a53bf86b0e2c) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.065119017Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
183992023-09-22T23:32:32.736Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.06518301Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184002023-09-22T23:32:32.739Z{"msg":"[1] client is_active_req TRUE, promote! session 38f823bf-6efb-41ed-8dda-a53bf86b0e2c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.065245245Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184012023-09-22T23:32:32.741Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.065740627Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184022023-09-22T23:32:32.744Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.065815103Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184032023-09-22T23:32:32.746Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.065904643Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184042023-09-22T23:32:32.749Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.065962788Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184052023-09-22T23:32:32.752Z{"msg":"9a30a600-1e9c-4c70-a202-c7991d534ac0 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.066034109Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184062023-09-22T23:32:32.754Z{"msg":"9a30a600-1e9c-4c70-a202-c7991d534ac0 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.066106848Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184072023-09-22T23:32:32.757Z{"msg":"[1] 9a30a600-1e9c-4c70-a202-c7991d534ac0 (38f823bf-6efb-41ed-8dda-a53bf86b0e2c) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.068078277Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184082023-09-22T23:32:32.759Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.068139341Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184092023-09-22T23:32:32.762Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:32.068181294Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184102023-09-22T23:32:32.765Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.068219998Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184112023-09-22T23:32:32.768Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.068258824Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184122023-09-22T23:32:32.771Z{"msg":"9a30a600-1e9c-4c70-a202-c7991d534ac0 WaitActive WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.06829781Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184132023-09-22T23:32:32.773Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.068336764Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184142023-09-22T23:32:32.776Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.068357002Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184152023-09-22T23:32:32.779Z{"msg":"[0] 9a30a600-1e9c-4c70-a202-c7991d534ac0 (38f823bf-6efb-41ed-8dda-a53bf86b0e2c) WaitActive WaitQuorum New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.068377771Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184162023-09-22T23:32:32.781Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.06839625Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184172023-09-22T23:32:32.784Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:32.068414225Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184182023-09-22T23:32:32.786Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.068432144Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184192023-09-22T23:32:32.789Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.068450002Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184202023-09-22T23:32:32.792Z{"msg":"9a30a600-1e9c-4c70-a202-c7991d534ac0 WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.068468455Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184212023-09-22T23:32:32.795Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.068486084Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184222023-09-22T23:32:32.798Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:32.068503363Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184232023-09-22T23:32:33.070Z{"msg":"[2] 9a30a600-1e9c-4c70-a202-c7991d534ac0 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.065450166Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737,"looper":"2"}
184242023-09-22T23:32:33.076Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.065532218Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184252023-09-22T23:32:33.084Z{"msg":"[2] 9a30a600-1e9c-4c70-a202-c7991d534ac0 (38f823bf-6efb-41ed-8dda-a53bf86b0e2c) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.066781596Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184262023-09-22T23:32:33.088Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.066837255Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184272023-09-22T23:32:33.094Z{"msg":"[2] client is_active_req TRUE, promote! session 38f823bf-6efb-41ed-8dda-a53bf86b0e2c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.066863263Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184282023-09-22T23:32:33.100Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.067462476Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184292023-09-22T23:32:33.107Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.067519399Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184302023-09-22T23:32:33.111Z{"msg":"9a30a600-1e9c-4c70-a202-c7991d534ac0 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.06756174Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184312023-09-22T23:32:33.114Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.069916053Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184322023-09-22T23:32:33.118Z{"msg":"[2] 9a30a600-1e9c-4c70-a202-c7991d534ac0 (38f823bf-6efb-41ed-8dda-a53bf86b0e2c) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.070033299Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184332023-09-22T23:32:33.122Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.070063515Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184342023-09-22T23:32:33.124Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:33.070112383Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184352023-09-22T23:32:33.127Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.07013458Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184362023-09-22T23:32:33.130Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.070173002Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184372023-09-22T23:32:33.133Z{"msg":"9a30a600-1e9c-4c70-a202-c7991d534ac0 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.070214512Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184382023-09-22T23:32:33.136Z{"msg":"[0]R flush_numbers[0..12]: [183, 202, 197, 205, 211, 205, 204, 209, 210, 194, 211, 205]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.070255241Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184392023-09-22T23:32:33.138Z{"msg":"[0]R generation[0..12]: [121, 132, 130, 135, 139, 135, 134, 137, 138, 128, 139, 135]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.070295433Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184402023-09-22T23:32:33.141Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.070335955Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184412023-09-22T23:32:33.144Z{"msg":"[1]R flush_numbers[0..12]: [183, 202, 197, 205, 211, 205, 204, 209, 210, 194, 211, 205]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.070357243Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184422023-09-22T23:32:33.147Z{"msg":"[1]R generation[0..12]: [121, 132, 130, 135, 139, 135, 134, 137, 138, 128, 139, 135]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.070376345Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184432023-09-22T23:32:33.150ZThe guest has finished waiting for activation with:140
184442023-09-22T23:32:33.153Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]Wait for a query_work_queue command to finish before sending IO
184452023-09-22T23:32:33.156Z","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.070401954Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184462023-09-22T23:32:33.159Z{"msg":"[2]R flush_numbers[0..12]: [183, 202, 197, 205, 211, 205, 204, 209, 210, 194, 211, 205]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.070421478Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184472023-09-22T23:32:33.162Z{"msg":"[2]R generation[0..12]: [121, 132, 130, 135, 139, 135, 134, 137, 138, 128, 139, 135]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.070440267Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184482023-09-22T23:32:33.165Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.070459459Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184492023-09-22T23:32:33.168Z{"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.070483427Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184502023-09-22T23:32:33.171Z{"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.070515867Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184512023-09-22T23:32:33.174Z{"msg":"Next flush: 212","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.070566804Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184522023-09-22T23:32:33.177Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.070608956Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184532023-09-22T23:32:33.180Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.070648524Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184542023-09-22T23:32:33.182Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.070686945Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184552023-09-22T23:32:33.185Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.070725737Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184562023-09-22T23:32:33.188Z{"msg":"9a30a600-1e9c-4c70-a202-c7991d534ac0 is now active with session: 38f823bf-6efb-41ed-8dda-a53bf86b0e2c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.070832637Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184572023-09-22T23:32:33.191Z{"msg":"9a30a600-1e9c-4c70-a202-c7991d534ac0 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.070873066Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184582023-09-22T23:32:33.194Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.070919261Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184592023-09-22T23:32:33.197Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.070984694Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184602023-09-22T23:32:33.199Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.071049909Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184612023-09-22T23:32:33.202Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.071112011Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184622023-09-22T23:32:33.205Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.071154322Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184632023-09-22T23:32:33.208Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.071197266Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184642023-09-22T23:32:33.211Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.071216826Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184652023-09-22T23:32:33.214Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.07123391Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184662023-09-22T23:32:33.217Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.071286181Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184672023-09-22T23:32:33.220Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.071352425Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184682023-09-22T23:32:33.222Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.071419159Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1737}
184692023-09-22T23:32:33.225ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
184702023-09-22T23:32:33.228ZLoading write count information from file "/tmp/test_repair_verify.data"
184712023-09-22T23:32:33.231ZRun Repair workload
184722023-09-22T23:32:33.234Z01/30 Write block 157 len 32768 data: 10 12 15 14 14 14 12 11
184732023-09-22T23:32:33.237Z02/30 Flush
184742023-09-22T23:32:33.240Z03/30 Flush
184752023-09-22T23:32:33.243Z04/30 Write block 433 len 28672 data: 14 12 15 14 13 15 13
184762023-09-22T23:32:33.245Z05/30 Read block 60 len 28672
184772023-09-22T23:32:33.248Z06/30 Flush
184782023-09-22T23:32:33.251Z07/30 Write block 91 len 20480 data: 12 12 12 11 13
184792023-09-22T23:32:33.254Z08/30 Read block 529 len 24576
184802023-09-22T23:32:33.257Z09/30 Write block 189 len 20480 data: 9 11 11 9 9
184812023-09-22T23:32:33.260Z10/30 Read block 418 len 40960
184822023-09-22T23:32:33.263Z11/30 Write block 152 len 32768 data: 10 7 9 10 9 11 13 16
184832023-09-22T23:32:33.265Z12/30 Write block 336 len 32768 data: 16 12 11 10 9 7 11 9
184842023-09-22T23:32:33.268Z13/30 Read block 451 len 28672
184852023-09-22T23:32:33.272Z14/30 Read block 408 len 20480
184862023-09-22T23:32:33.275Z15/30 Write block 72 len 16384 data: 12 9 10 11
184872023-09-22T23:32:33.279Z16/30 Write block 467 len 32768 data: 12 13 13 10 8 9 8 8
184882023-09-22T23:32:33.284Z17/30 Flush
184892023-09-22T23:32:33.287Z18/30 Write block 292 len 36864 data: 19 16 15 16 15 11 11 11 9
184902023-09-22T23:32:33.290Z19/30 Read block 201 len 16384
184912023-09-22T23:32:33.292Z20/30 Write block 72 len 20480 data: 13 10 11 12 11
184922023-09-22T23:32:33.295Z21/30 Read block 420 len 36864
184932023-09-22T23:32:33.299Z22/30 Read block 87 len 28672
184942023-09-22T23:32:33.302Z23/30 Write block 56 len 40960 data: 4 5 9 10 14 14 16 19 18 21
184952023-09-22T23:32:33.305Z24/30 Read block 577 len 16384
184962023-09-22T23:32:33.308Z25/30 Write block 450 len 28672 data: 13 12 11 13 11 11 10
184972023-09-22T23:32:33.312Z26/30 Flush
184982023-09-22T23:32:33.315Z27/30 Write block 41 len 36864 data: 11 10 9 9 10 9 10 9 9
184992023-09-22T23:32:33.318Z28/30 Read block 155 len 12288
185002023-09-22T23:32:33.320Z29/30 Write block 311 len 16384 data: 6 7 9 11
185012023-09-22T23:32:33.323Z30/30 Read block 146 len 36864
185022023-09-22T23:32:33.326Z----------------------------------------------------------------
185032023-09-22T23:32:33.329Z Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:28
185042023-09-22T23:32:33.332ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
185052023-09-22T23:32:33.337Z 3 Acked 1002 Flush 0 Done Done Sent false
185062023-09-22T23:32:33.340Z 4 Acked 1003 Write 7 Done Done Sent false
185072023-09-22T23:32:33.343Z 5 Acked 1004 Read 7 Done Done Sent false
185082023-09-22T23:32:33.346Z 6 Acked 1005 Flush 0 Done Done Sent false
185092023-09-22T23:32:33.349Z 7 Acked 1006 Write 5 Done Done Sent false
185102023-09-22T23:32:33.353Z 8 Acked 1007 Read 6 Done Done Sent false
185112023-09-22T23:32:33.356Z 9 Acked 1008 Write 5 Done Done Sent false
185122023-09-22T23:32:33.359Z 10 Acked 1009 Read 10 Done Done Sent false
185132023-09-22T23:32:33.363Z 11 Acked 1010 Write 8 Done Done Sent false
185142023-09-22T23:32:33.367Z 12 Acked 1011 Write 8 Done Done Sent false
185152023-09-22T23:32:33.370Z 13 Acked 1012 Read 7 Done Done Sent false
185162023-09-22T23:32:33.373Z 14 Acked 1013 Read 5 Done Done Sent false
185172023-09-22T23:32:33.376Z 15 Acked 1014 Write 4 Done Done Sent false
185182023-09-22T23:32:33.379Z 16 Acked 1015 Write 8 Done Done Sent false
185192023-09-22T23:32:33.382Z 17 Acked 1016 Flush 0 Done Done Sent false
185202023-09-22T23:32:33.385Z 18 Acked 1017 Write 9 Done Done Sent false
185212023-09-22T23:32:33.389Z 19 Acked 1018 Read 4 Done Done Sent false
185222023-09-22T23:32:33.392Z 20 Acked 1019 Write 5 Done Done Sent false
185232023-09-22T23:32:33.395Z 21 Acked 1020 Read 9 Done Done Sent false
185242023-09-22T23:32:33.402Z 22 Acked 1021 Read 7 Done Done Sent false
185252023-09-22T23:32:33.412Z 23 Acked 1022 Write 10 Done Done Sent false
185262023-09-22T23:32:33.421Z 24 Acked 1023 Read 4 Done Done Sent false
185272023-09-22T23:32:33.442Z 25 Acked 1024 Write 7 Done Done Sent false
185282023-09-22T23:32:33.458Z 26 Acked 1025 Flush 0 Done Done Sent false
185292023-09-22T23:32:33.466Z 27 Acked 1026 Write 9 Done Done Sent false
185302023-09-22T23:32:33.474Z 28 Acked 1027 Read 3 Done Done Sent false
185312023-09-22T23:32:33.486Z 29 Acked 1028 Write 4 Done Done Sent false
185322023-09-22T23:32:33.494Z 30 Acked 1029 Read 9 Done Done Sent false
185332023-09-22T23:32:33.510Z STATES DS:0 DS:1 DS:2 TOTAL
185342023-09-22T23:32:33.528Z New 0 0 0 0
185352023-09-22T23:32:33.537Z Sent 0 0 28 28
185362023-09-22T23:32:33.543Z Done 28 28 0 56
185372023-09-22T23:32:33.550Z Skipped 0 0 0 0
185382023-09-22T23:32:33.554Z Error 0 0 0 0
185392023-09-22T23:32:33.557ZLast Flush: 1025 1025 1001
185402023-09-22T23:32:33.561ZDownstairs last five completed: 1001 1000
185412023-09-22T23:32:33.564ZUpstairs last five completed: 30 29 28 27 26 25
185422023-09-22T23:32:33.568ZWrote out file "/tmp/test_repair_verify.data"
185432023-09-22T23:32:33.570Z
185442023-09-22T23:32:33.573ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
185452023-09-22T23:32:33.576Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.227621525Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1738}
185462023-09-22T23:32:33.579Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.228009875Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1738}
185472023-09-22T23:32:33.581Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.22810572Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1738}
185482023-09-22T23:32:33.584Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.250384665Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1738}
185492023-09-22T23:32:33.587Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.250416373Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1738}
185502023-09-22T23:32:33.590Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.250435592Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1738}
185512023-09-22T23:32:33.592Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.264332446Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1738}
185522023-09-22T23:32:33.595Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.264362943Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1738}
185532023-09-22T23:32:33.598Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:33.264381396Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1738}
185542023-09-22T23:32:33.601ZError: Difference in extent metadata found!
185552023-09-22T23:32:33.603ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
185562023-09-22T23:32:33.606Z 0 000-019 121 121 121 183 183 183 F F F
185572023-09-22T23:32:33.609Z 1 020-039 132 132 132 202 202 202 F F F
185582023-09-22T23:32:33.612Z 2 040-059 140 140 130 216 216 197 T T F <---
185592023-09-22T23:32:33.615Z 3 060-079 140 140 135 216 216 205 F F F <---
185602023-09-22T23:32:33.620Z 4 080-099 140 140 139 215 215 211 F F F <---
185612023-09-22T23:32:33.623Z 5 100-119 135 135 135 205 205 205 F F F
185622023-09-22T23:32:33.626Z 6 120-139 134 134 134 204 204 204 F F F
185632023-09-22T23:32:33.630Z 7 140-159 140 140 140 215 215 212 F F F <---
185642023-09-22T23:32:33.632Z 8 160-179 140 140 140 212 212 212 F F F
185652023-09-22T23:32:33.635Z 9 180-199 140 140 128 215 215 194 F F F <---
185662023-09-22T23:32:33.638Z 10 200-219 139 139 139 211 211 211 F F F
185672023-09-22T23:32:33.641Z 11 220-239 135 135 135 205 205 205 F F F
185682023-09-22T23:32:33.644Z 12 240-259 139 139 139 211 211 211 F F F
185692023-09-22T23:32:33.646Z 13 260-279 139 139 139 211 211 211 F F F
185702023-09-22T23:32:33.649Z 14 280-299 140 140 137 216 216 209 F F F <---
185712023-09-22T23:32:33.652Z 15 300-319 140 140 137 216 216 209 T T F <---
185722023-09-22T23:32:33.655Z 16 320-339 140 140 139 215 215 211 F F F <---
185732023-09-22T23:32:33.658Z 17 340-359 140 140 137 215 215 209 F F F <---
185742023-09-22T23:32:33.661Z 18 360-379 134 134 134 204 204 204 F F F
185752023-09-22T23:32:33.664Z 19 380-399 135 135 135 205 205 205 F F F
185762023-09-22T23:32:33.666Z 20 400-419 135 135 135 205 205 205 F F F
185772023-09-22T23:32:33.669Z 21 420-439 140 140 138 214 214 210 F F F <---
185782023-09-22T23:32:33.672Z 22 440-459 140 140 138 216 216 210 F F F <---
185792023-09-22T23:32:33.675Z 23 460-479 140 140 134 215 215 204 F F F <---
185802023-09-22T23:32:33.677Z 24 480-499 137 137 137 209 209 209 F F F
185812023-09-22T23:32:33.680Z 25 500-519 136 136 136 207 207 207 F F F
185822023-09-22T23:32:33.683Z 26 520-539 139 139 139 211 211 211 F F F
185832023-09-22T23:32:33.706Z 27 540-559 136 136 136 208 208 208 F F F
185842023-09-22T23:32:33.709Z 28 560-579 138 138 138 210 210 210 F F F
185852023-09-22T23:32:33.712Z 29 580-599 138 138 138 210 210 210 F F F
185862023-09-22T23:32:33.715ZMax gen: 140, Max flush: 216
185872023-09-22T23:32:33.718ZA Difference in extent metadata is expected here
185882023-09-22T23:32:33.721ZOn loop 70
185892023-09-22T23:32:33.724Z
185902023-09-22T23:32:33.727ZVerifying data now
185912023-09-22T23:32:39.600ZNo change after verify
185922023-09-22T23:32:39.614ZLoop: 70 Downstairs dump after verify (and repair):
185932023-09-22T23:32:39.632Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.612555147Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1743}
185942023-09-22T23:32:39.642Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.612916026Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1743}
185952023-09-22T23:32:39.654Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.612959202Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1743}
185962023-09-22T23:32:39.665Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.631738579Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1743}
185972023-09-22T23:32:39.676Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.631787809Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1743}
185982023-09-22T23:32:39.688Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.631825258Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1743}
185992023-09-22T23:32:39.697Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.642756538Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1743}
186002023-09-22T23:32:39.705Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.642801473Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1743}
186012023-09-22T23:32:39.713Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.642838645Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1743}
186022023-09-22T23:32:39.720ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
186032023-09-22T23:32:39.734Z 0 000-019 121 121 121 183 183 183 F F F
186042023-09-22T23:32:39.739Z 1 020-039 132 132 132 202 202 202 F F F
186052023-09-22T23:32:39.742Z 2 040-059 141 141 141 217 217 217 F F F
186062023-09-22T23:32:39.744Z 3 060-079 140 140 140 216 216 216 F F F
186072023-09-22T23:32:39.749Z 4 080-099 140 140 140 215 215 215 F F F
186082023-09-22T23:32:39.752Z 5 100-119 135 135 135 205 205 205 F F F
186092023-09-22T23:32:39.755Z 6 120-139 134 134 134 204 204 204 F F F
186102023-09-22T23:32:39.760Z 7 140-159 140 140 140 215 215 215 F F F
186112023-09-22T23:32:39.763Z 8 160-179 140 140 140 212 212 212 F F F
186122023-09-22T23:32:39.766Z 9 180-199 140 140 140 215 215 215 F F F
186132023-09-22T23:32:39.769Z 10 200-219 139 139 139 211 211 211 F F F
186142023-09-22T23:32:39.772Z 11 220-239 135 135 135 205 205 205 F F F
186152023-09-22T23:32:39.776Z 12 240-259 139 139 139 211 211 211 F F F
186162023-09-22T23:32:39.779Z 13 260-279 139 139 139 211 211 211 F F F
186172023-09-22T23:32:39.782Z 14 280-299 140 140 140 216 216 216 F F F
186182023-09-22T23:32:39.785Z 15 300-319 141 141 141 217 217 217 F F F
186192023-09-22T23:32:39.788Z 16 320-339 140 140 140 215 215 215 F F F
186202023-09-22T23:32:39.791Z 17 340-359 140 140 140 215 215 215 F F F
186212023-09-22T23:32:39.794Z 18 360-379 134 134 134 204 204 204 F F F
186222023-09-22T23:32:39.797Z 19 380-399 135 135 135 205 205 205 F F F
186232023-09-22T23:32:39.800Z 20 400-419 135 135 135 205 205 205 F F F
186242023-09-22T23:32:39.819Z 21 420-439 140 140 140 214 214 214 F F F
186252023-09-22T23:32:40.232Z 22 440-459 140 140 140 216 216 216 F F F
186262023-09-22T23:32:40.236Z 23 460-479 140 140 140 215 215 215 F F F
186272023-09-22T23:32:40.240Z 24 480-499 137 137 137 209 209 209 F F F
186282023-09-22T23:32:40.251Z 25 500-519 136 136 136 207 207 207 F F F
186292023-09-22T23:32:40.258Z 26 520-539 139 139 139 211 211 211 F F F
186302023-09-22T23:32:40.266Z 27 540-559 136 136 136 208 208 208 F F F
186312023-09-22T23:32:40.273Z 28 560-579 138 138 138 210 210 210 F F F
186322023-09-22T23:32:40.280Z 29 580-599 138 138 138 210 210 210 F F F
186332023-09-22T23:32:40.285ZMax gen: 141, Max flush: 217
186342023-09-22T23:32:40.288Z
186352023-09-22T23:32:40.291ZBegin loop 71
186362023-09-22T23:32:40.294ZDownstairs to restart: 1
186372023-09-22T23:32:40.297Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.690877826Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186382023-09-22T23:32:40.299Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.691252874Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186392023-09-22T23:32:40.302Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.691313803Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186402023-09-22T23:32:40.305Z{"msg":"Crucible stats registered with UUID: 79e271ef-cd9b-4d26-a390-7aa2d1596cde","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.69133779Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186412023-09-22T23:32:40.308Z{"msg":"Crucible 79e271ef-cd9b-4d26-a390-7aa2d1596cde has session id: 056aa250-f981-443a-874a-05d6ca77ff14","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.691386469Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186422023-09-22T23:32:40.311ZCrucible runtime is spawned
186432023-09-22T23:32:40.314ZThe guest has requested activation with gen:142
186442023-09-22T23:32:40.316Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.691761241Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745,"task":"up_listen"}
186452023-09-22T23:32:40.319Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.691853745Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186462023-09-22T23:32:40.322Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.691935625Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186472023-09-22T23:32:40.325Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.691979226Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745,"looper":"2"}
186482023-09-22T23:32:40.328Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.69209869Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745,"looper":"0"}
186492023-09-22T23:32:40.331Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.692175492Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745,"looper":"1"}
186502023-09-22T23:32:40.334Z{"msg":"[2] 79e271ef-cd9b-4d26-a390-7aa2d1596cde looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.692298287Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745,"looper":"2"}
186512023-09-22T23:32:40.343Z{"msg":"[0] 79e271ef-cd9b-4d26-a390-7aa2d1596cde looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.692368377Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745,"looper":"0"}
186522023-09-22T23:32:40.354Z{"msg":"79e271ef-cd9b-4d26-a390-7aa2d1596cde active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.692617077Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186532023-09-22T23:32:40.364Z{"msg":"Set desired generation to :142","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.692674539Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186542023-09-22T23:32:40.378Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.692758306Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186552023-09-22T23:32:40.388Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.693051999Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186562023-09-22T23:32:40.399Z{"msg":"[2] 79e271ef-cd9b-4d26-a390-7aa2d1596cde (c394da41-c0fb-4527-8647-f77e417ff958) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.693378089Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186572023-09-22T23:32:40.415Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.693462111Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186582023-09-22T23:32:40.425Z{"msg":"[2] client is_active_req TRUE, promote! session c394da41-c0fb-4527-8647-f77e417ff958","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.69350859Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186592023-09-22T23:32:40.849Z{"msg":"[0] 79e271ef-cd9b-4d26-a390-7aa2d1596cde (c394da41-c0fb-4527-8647-f77e417ff958) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.693542422Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186602023-09-22T23:32:40.853Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.693588619Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186612023-09-22T23:32:40.856Z{"msg":"[0] client is_active_req TRUE, promote! session c394da41-c0fb-4527-8647-f77e417ff958","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.693609873Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186622023-09-22T23:32:40.858Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.694027764Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186632023-09-22T23:32:40.865Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.69409635Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186642023-09-22T23:32:40.868Z{"msg":"79e271ef-cd9b-4d26-a390-7aa2d1596cde WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.694187206Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186652023-09-22T23:32:40.871Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.694251975Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186662023-09-22T23:32:40.874Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.694280609Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186672023-09-22T23:32:40.877Z{"msg":"79e271ef-cd9b-4d26-a390-7aa2d1596cde WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.694361978Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186682023-09-22T23:32:40.880Z{"msg":"[2] 79e271ef-cd9b-4d26-a390-7aa2d1596cde (c394da41-c0fb-4527-8647-f77e417ff958) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.696831002Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186692023-09-22T23:32:40.883Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.696931744Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186702023-09-22T23:32:40.886Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:39.696970689Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186712023-09-22T23:32:40.889Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.697002075Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186722023-09-22T23:32:40.891Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.697022239Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186732023-09-22T23:32:40.894Z{"msg":"79e271ef-cd9b-4d26-a390-7aa2d1596cde WaitActive New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.69704053Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186742023-09-22T23:32:40.897Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.697058617Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186752023-09-22T23:32:40.900Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.697076024Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186762023-09-22T23:32:40.903Z{"msg":"[0] 79e271ef-cd9b-4d26-a390-7aa2d1596cde (c394da41-c0fb-4527-8647-f77e417ff958) WaitActive New WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.697188914Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186772023-09-22T23:32:40.906Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.697216499Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186782023-09-22T23:32:40.909Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:39.697234797Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186792023-09-22T23:32:40.911Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.697252457Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186802023-09-22T23:32:40.934Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.697270705Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186812023-09-22T23:32:40.938Z{"msg":"79e271ef-cd9b-4d26-a390-7aa2d1596cde WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.69728886Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186822023-09-22T23:32:40.941Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.697306526Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186832023-09-22T23:32:40.943Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:39.697324049Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186842023-09-22T23:32:40.946Z{"msg":"[1] 79e271ef-cd9b-4d26-a390-7aa2d1596cde looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.694499364Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745,"looper":"1"}
186852023-09-22T23:32:40.949Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.6945967Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186862023-09-22T23:32:40.951Z{"msg":"[1] 79e271ef-cd9b-4d26-a390-7aa2d1596cde (c394da41-c0fb-4527-8647-f77e417ff958) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.695717978Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186872023-09-22T23:32:40.954Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.695775277Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186882023-09-22T23:32:40.957Z{"msg":"[1] client is_active_req TRUE, promote! session c394da41-c0fb-4527-8647-f77e417ff958","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.695818006Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186892023-09-22T23:32:40.959Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.69638249Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186902023-09-22T23:32:40.962Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.696448227Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186912023-09-22T23:32:40.965Z{"msg":"79e271ef-cd9b-4d26-a390-7aa2d1596cde WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.696493408Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186922023-09-22T23:32:40.969Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.698630148Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186932023-09-22T23:32:40.972Z{"msg":"[1] 79e271ef-cd9b-4d26-a390-7aa2d1596cde (c394da41-c0fb-4527-8647-f77e417ff958) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.698870846Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186942023-09-22T23:32:40.975Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.698925534Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186952023-09-22T23:32:40.978Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:40.698967981Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186962023-09-22T23:32:40.981Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.698989754Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186972023-09-22T23:32:40.984Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.699027387Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186982023-09-22T23:32:40.986Z{"msg":"79e271ef-cd9b-4d26-a390-7aa2d1596cde WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.699050127Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
186992023-09-22T23:32:40.990Z{"msg":"[0]R flush_numbers[0..12]: [183, 202, 217, 216, 215, 205, 204, 215, 212, 215, 211, 205]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.699088398Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187002023-09-22T23:32:40.993Z{"msg":"[0]R generation[0..12]: [121, 132, 141, 140, 140, 135, 134, 140, 140, 140, 139, 135]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.69912799Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187012023-09-22T23:32:40.996Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.699236868Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745The guest has finished waiting for activation with:142
187022023-09-22T23:32:40.999ZWait for a query_work_queue command to finish before sending IO
187032023-09-22T23:32:41.003Z}
187042023-09-22T23:32:41.006Z{"msg":"[1]R flush_numbers[0..12]: [183, 202, 217, 216, 215, 205, 204, 215, 212, 215, 211, 205]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.699447328Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187052023-09-22T23:32:41.009Z{"msg":"[1]R generation[0..12]: [121, 132, 141, 140, 140, 135, 134, 140, 140, 140, 139, 135]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.699522418Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187062023-09-22T23:32:41.012Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.699601621Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187072023-09-22T23:32:41.015Z{"msg":"[2]R flush_numbers[0..12]: [183, 202, 217, 216, 215, 205, 204, 215, 212, 215, 211, 205]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.699653611Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187082023-09-22T23:32:41.018Z{"msg":"[2]R generation[0..12]: [121, 132, 141, 140, 140, 135, 134, 140, 140, 140, 139, 135]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.69969494Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187092023-09-22T23:32:41.021Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.699751921Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187102023-09-22T23:32:41.024Z{"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.699801428Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187112023-09-22T23:32:41.026Z{"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.69984182Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187122023-09-22T23:32:41.029Z{"msg":"Next flush: 218","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.699888912Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187132023-09-22T23:32:41.032Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.699944525Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187142023-09-22T23:32:41.034Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.699989037Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187152023-09-22T23:32:41.037Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.700029421Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187162023-09-22T23:32:41.040Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.700074408Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187172023-09-22T23:32:41.043Z{"msg":"79e271ef-cd9b-4d26-a390-7aa2d1596cde is now active with session: c394da41-c0fb-4527-8647-f77e417ff958","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.700136179Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187182023-09-22T23:32:41.045Z{"msg":"79e271ef-cd9b-4d26-a390-7aa2d1596cde Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.700179223Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187192023-09-22T23:32:41.048Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.700200902Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187202023-09-22T23:32:41.051Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.700262125Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187212023-09-22T23:32:41.054Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
187222023-09-22T23:32:41.057Z,"time":"2023-09-22T23:32:40.700323467Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187232023-09-22T23:32:41.060Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.700439745Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187242023-09-22T23:32:41.062Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.700486071Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187252023-09-22T23:32:41.065Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.700524762Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187262023-09-22T23:32:41.068Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.700544803Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187272023-09-22T23:32:41.071Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.700561739Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187282023-09-22T23:32:41.074Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.70057875Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187292023-09-22T23:32:41.077Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.700596439Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187302023-09-22T23:32:41.080Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.700613378Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1745}
187312023-09-22T23:32:41.082ZLoading write count information from file "/tmp/test_repair_verify.data"
187322023-09-22T23:32:41.085ZRun Repair workload
187332023-09-22T23:32:41.087Z01/30 Write block 327 len 16384 data: 7 11 11 8
187342023-09-22T23:32:41.090Z02/30 Flush
187352023-09-22T23:32:41.093Z03/30 Write block 72 len 40960 data: 14 11 12 13 12 13 13 16 14 15
187362023-09-22T23:32:41.096Z04/30 Write block 588 len 12288 data: 8 9 9
187372023-09-22T23:32:41.099Z05/30 Write block 418 len 36864 data: 13 14 11 11 9 9 9 6 9
187382023-09-22T23:32:41.102Z06/30 Read block 436 len 16384
187392023-09-22T23:32:41.105Z07/30 Read block 176 len 28672
187402023-09-22T23:32:41.107Z08/30 Read block 352 len 36864
187412023-09-22T23:32:41.110Z09/30 Read block 285 len 16384
187422023-09-22T23:32:41.113Z10/30 Read block 65 len 16384
187432023-09-22T23:32:41.116Z11/30 Write block 587 len 40960 data: 7 9 10 10 10 12 14 11 11 11
187442023-09-22T23:32:41.119Z12/30 Read block 336 len 12288
187452023-09-22T23:32:41.122Z13/30 Flush
187462023-09-22T23:32:41.125Z14/30 Write block 233 len 40960 data: 7 8 9 10 10 10 12 14 14 12
187472023-09-22T23:32:41.129ZError: Difference in extent metadata found!
187482023-09-22T23:32:41.134Z15/30 Write block 123 len 28672 data: 13 11 9 7 11 11 9
187492023-09-22T23:32:41.138Z16/30 Write block 102 len 36864 data: 16 18 21 23 21 20 20 17 17
187502023-09-22T23:32:41.140Z17/30 Write block 204 len 4096 data: 15
187512023-09-22T23:32:41.143Z18/30 Flush
187522023-09-22T23:32:41.146Z19/30 Write block 369 len 12288 data: 8 7 8
187532023-09-22T23:32:41.149Z20/30 Read block 4 len 32768
187542023-09-22T23:32:41.152Z21/30 Read block 473 len 24576
187552023-09-22T23:32:41.156Z22/30 Read block 176 len 12288
187562023-09-22T23:32:41.170Z23/30 Flush
187572023-09-22T23:32:41.196Z24/30 Write block 109 len 32768 data: 18 18 13 13 13 11 10 10
187582023-09-22T23:32:41.207Z25/30 Write block 575 len 24576 data: 7 8 10 11 9 10
187592023-09-22T23:32:41.217Z26/30 Read block 36 len 28672
187602023-09-22T23:32:41.226Z27/30 Write block 13 len 12288 data: 11 9 10
187612023-09-22T23:32:41.232Z28/30 Write block 361 len 4096 data: 12
187622023-09-22T23:32:41.236Z29/30 Write block 502 len 12288 data: 10 9 10
187632023-09-22T23:32:41.239Z30/30 Read block 181 len 12288
187642023-09-22T23:32:41.242Z----------------------------------------------------------------
187652023-09-22T23:32:41.245Z Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:28
187662023-09-22T23:32:41.248ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
187672023-09-22T23:32:41.250Z 3 Acked 1002 Write 10 Done Done Done false
187682023-09-22T23:32:41.253Z 4 Acked 1003 Write 3 Done Done Done false
187692023-09-22T23:32:41.256Z 5 Acked 1004 Write 9 Done Done Done false
187702023-09-22T23:32:41.259Z 6 Acked 1005 Read 4 Done Done Done false
187712023-09-22T23:32:41.262Z 7 Acked 1006 Read 7 Done Done Done false
187722023-09-22T23:32:41.265Z 8 Acked 1007 Read 9 Done Done Done false
187732023-09-22T23:32:41.267Z 9 Acked 1008 Read 4 Done Done Done false
187742023-09-22T23:32:41.270Z 10 Acked 1009 Read 4 Done Done Done false
187752023-09-22T23:32:41.273Z 11 Acked 1010 Write 10 Done Done Done false
187762023-09-22T23:32:41.276Z 12 Acked 1011 Read 3 Done Done Done false
187772023-09-22T23:32:41.279Z 13 Acked 1012 Flush 0 Done Sent Done false
187782023-09-22T23:32:41.282Z 14 Acked 1013 Write 10 Done Sent Done false
187792023-09-22T23:32:41.284Z 15 Acked 1014 Write 7 Done Sent Done false
187802023-09-22T23:32:41.287Z 16 Acked 1015 Write 9 Done Sent Done false
187812023-09-22T23:32:41.290Z 17 Acked 1016 Write 1 Done Sent Done false
187822023-09-22T23:32:41.293Z 18 Acked 1017 Flush 0 Done Sent Done false
187832023-09-22T23:32:41.296Z 19 Acked 1018 Write 3 Done Sent Done false
187842023-09-22T23:32:41.298Z 20 Acked 1019 Read 8 Done Sent Done false
187852023-09-22T23:32:41.301Z 21 Acked 1020 Read 6 Done Sent Done false
187862023-09-22T23:32:41.304Z 22 Acked 1021 Read 3 Done Sent Done false
187872023-09-22T23:32:41.307Z 23 Acked 1022 Flush 0 Done Sent Done false
187882023-09-22T23:32:41.310Z 24 Acked 1023 Write 8 Done Sent Done false
187892023-09-22T23:32:41.313Z 25 Acked 1024 Write 6 Done Sent Done false
187902023-09-22T23:32:41.316Z 26 Acked 1025 Read 7 Done Sent Done false
187912023-09-22T23:32:41.318Z 27 Acked 1026 Write 3 Done Sent Done false
187922023-09-22T23:32:41.321Z 28 Acked 1027 Write 1 Done Sent Done false
187932023-09-22T23:32:41.324Z 29 Acked 1028 Write 3 Done Sent Sent false
187942023-09-22T23:32:41.327Z 30 Acked 1029 Read 3 Done Sent Sent false
187952023-09-22T23:32:41.330Z STATES DS:0 DS:1 DS:2 TOTAL
187962023-09-22T23:32:41.332Z New 0 0 0 0
187972023-09-22T23:32:41.335Z Sent 0 18 2 20
187982023-09-22T23:32:41.338Z Done 28 10 26 64
187992023-09-22T23:32:41.341Z Skipped 0 0 0 0
188002023-09-22T23:32:41.343Z Error 0 0 0 0
188012023-09-22T23:32:41.346ZLast Flush: 1022 1001 1022
188022023-09-22T23:32:41.349ZDownstairs last five completed: 1001 1000
188032023-09-22T23:32:41.352ZUpstairs last five completed: 30 29 28 27 26 25
188042023-09-22T23:32:41.355ZWrote out file "/tmp/test_repair_verify.data"
188052023-09-22T23:32:41.358Z
188062023-09-22T23:32:41.361ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
188072023-09-22T23:32:41.363Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.846641145Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1746}
188082023-09-22T23:32:41.366Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.846896449Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1746}
188092023-09-22T23:32:41.369Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.846919281Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1746}
188102023-09-22T23:32:41.372Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.876112225Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1746}
188112023-09-22T23:32:41.375Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.876145131Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1746}
188122023-09-22T23:32:41.378Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.876163636Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1746}
188132023-09-22T23:32:41.381Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.916412281Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1746}
188142023-09-22T23:32:41.383Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.916447603Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1746}
188152023-09-22T23:32:41.386Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:40.916466886Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1746}
188162023-09-22T23:32:41.389ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
188172023-09-22T23:32:41.392Z 0 000-019 121 121 121 183 183 183 T F T <---
188182023-09-22T23:32:41.395Z 1 020-039 132 132 132 202 202 202 F F F
188192023-09-22T23:32:41.399Z 2 040-059 141 141 141 217 217 217 F F F
188202023-09-22T23:32:41.401Z 3 060-079 142 140 142 219 216 219 F T F <---
188212023-09-22T23:32:41.404Z 4 080-099 142 140 142 219 215 219 F T F <---
188222023-09-22T23:32:41.407Z 5 100-119 142 135 142 220 205 220 T F T <---
188232023-09-22T23:32:41.410Z 6 120-139 142 134 142 220 204 220 F F F <---
188242023-09-22T23:32:41.413Z 7 140-159 140 140 140 215 215 215 F F F
188252023-09-22T23:32:41.415Z 8 160-179 140 140 140 212 212 212 F F F
188262023-09-22T23:32:41.418Z 9 180-199 140 140 140 215 215 215 F F F
188272023-09-22T23:32:41.421Z 10 200-219 142 139 142 220 211 220 F F F <---
188282023-09-22T23:32:41.423Z 11 220-239 142 135 142 220 205 220 F F F <---
188292023-09-22T23:32:41.426Z 12 240-259 142 139 142 220 211 220 F F F <---
188302023-09-22T23:32:41.429Z 13 260-279 139 139 139 211 211 211 F F F
188312023-09-22T23:32:41.432Z 14 280-299 140 140 140 216 216 216 F F F
188322023-09-22T23:32:41.435Z 15 300-319 141 141 141 217 217 217 F F F
188332023-09-22T23:32:41.437Z 16 320-339 142 142 142 218 218 218 F F F
188342023-09-22T23:32:41.440Z 17 340-359 140 140 140 215 215 215 F F F
188352023-09-22T23:32:41.443Z 18 360-379 142 134 142 221 204 221 T F T <---
188362023-09-22T23:32:41.445Z 19 380-399 135 135 135 205 205 205 F F F
188372023-09-22T23:32:41.448Z 20 400-419 142 135 142 219 205 219 F T F <---
188382023-09-22T23:32:41.451Z 21 420-439 142 140 142 219 214 219 F T F <---
188392023-09-22T23:32:41.454Z 22 440-459 140 140 140 216 216 216 F F F
188402023-09-22T23:32:41.457Z 23 460-479 140 140 140 215 215 215 F F F
188412023-09-22T23:32:41.460Z 24 480-499 137 137 137 209 209 209 F F F
188422023-09-22T23:32:41.463Z 25 500-519 136 136 136 207 207 207 T F T <---
188432023-09-22T23:32:41.466Z 26 520-539 139 139 139 211 211 211 F F F
188442023-09-22T23:32:41.469Z 27 540-559 136 136 136 208 208 208 F F F
188452023-09-22T23:32:41.471Z 28 560-579 138 138 138 210 210 210 T F T <---
188462023-09-22T23:32:41.474Z 29 580-599 142 138 142 219 210 219 T T T <---
188472023-09-22T23:32:41.477ZMax gen: 142, Max flush: 221
188482023-09-22T23:32:41.480ZA Difference in extent metadata is expected here
188492023-09-22T23:32:41.483ZOn loop 71
188502023-09-22T23:32:41.486Z
188512023-09-22T23:32:41.488ZVerifying data now
188522023-09-22T23:32:47.452ZNo change after verify
188532023-09-22T23:32:47.455ZLoop: 71 Downstairs dump after verify (and repair):
188542023-09-22T23:32:47.471Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.470060129Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1751}
188552023-09-22T23:32:47.474Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.470366186Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1751}
188562023-09-22T23:32:47.477Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.470390925Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1751}
188572023-09-22T23:32:47.490Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.489292651Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1751}
188582023-09-22T23:32:47.493Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.489344476Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1751}
188592023-09-22T23:32:47.496Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.489364779Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1751}
188602023-09-22T23:32:47.501Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.500565069Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1751}
188612023-09-22T23:32:47.504Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.500617363Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1751}
188622023-09-22T23:32:47.507Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.500637895Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1751}
188632023-09-22T23:32:47.512ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
188642023-09-22T23:32:47.515Z 0 000-019 143 143 143 222 222 222 F F F
188652023-09-22T23:32:47.518Z 1 020-039 132 132 132 202 202 202 F F F
188662023-09-22T23:32:47.521Z 2 040-059 141 141 141 217 217 217 F F F
188672023-09-22T23:32:47.525Z 3 060-079 142 142 142 219 219 219 F F F
188682023-09-22T23:32:47.527Z 4 080-099 142 142 142 219 219 219 F F F
188692023-09-22T23:32:47.530Z 5 100-119 143 143 143 222 222 222 F F F
188702023-09-22T23:32:47.533Z 6 120-139 142 142 142 220 220 220 F F F
188712023-09-22T23:32:47.536Z 7 140-159 140 140 140 215 215 215 F F F
188722023-09-22T23:32:47.539Z 8 160-179 140 140 140 212 212 212 F F F
188732023-09-22T23:32:47.541Z 9 180-199 140 140 140 215 215 215 F F F
188742023-09-22T23:32:47.544Z 10 200-219 142 142 142 220 220 220 F F F
188752023-09-22T23:32:47.547Z 11 220-239 142 142 142 220 220 220 F F F
188762023-09-22T23:32:47.550Z 12 240-259 142 142 142 220 220 220 F F F
188772023-09-22T23:32:47.553Z 13 260-279 139 139 139 211 211 211 F F F
188782023-09-22T23:32:47.555Z 14 280-299 140 140 140 216 216 216 F F F
188792023-09-22T23:32:47.558Z 15 300-319 141 141 141 217 217 217 F F F
188802023-09-22T23:32:47.561Z 16 320-339 142 142 142 218 218 218 F F F
188812023-09-22T23:32:47.564Z 17 340-359 140 140 140 215 215 215 F F F
188822023-09-22T23:32:47.569Z 18 360-379 143 143 143 222 222 222 F F F
188832023-09-22T23:32:47.573Z 19 380-399 135 135 135 205 205 205 F F F
188842023-09-22T23:32:47.576Z 20 400-419 142 142 142 219 219 219 F F F
188852023-09-22T23:32:47.578Z 21 420-439 142 142 142 219 219 219 F F F
188862023-09-22T23:32:47.581Z 22 440-459 140 140 140 216 216 216 F F F
188872023-09-22T23:32:47.584Z 23 460-479 140 140 140 215 215 215 F F F
188882023-09-22T23:32:47.587Z 24 480-499 137 137 137 209 209 209 F F F
188892023-09-22T23:32:47.589Z 25 500-519 143 143 143 222 222 222 F F F
188902023-09-22T23:32:47.592Z 26 520-539 139 139 139 211 211 211 F F F
188912023-09-22T23:32:47.595Z 27 540-559 136 136 136 208 208 208 F F F
188922023-09-22T23:32:47.598Z 28 560-579 143 143 143 222 222 222 F F F
188932023-09-22T23:32:47.601Z 29 580-599 143 143 143 222 222 222 F F F
188942023-09-22T23:32:47.604ZMax gen: 143, Max flush: 222
188952023-09-22T23:32:47.607Z
188962023-09-22T23:32:47.610ZBegin loop 72
188972023-09-22T23:32:47.612ZDownstairs to restart: 1
188982023-09-22T23:32:47.615Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.54763378Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
188992023-09-22T23:32:47.618Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.548002803Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189002023-09-22T23:32:47.621Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.54805966Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189012023-09-22T23:32:47.623Z{"msg":"Crucible stats registered with UUID: e5b3755f-5ec3-4c87-a0e0-2f3ce464924e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.548127867Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189022023-09-22T23:32:47.626Z{"msg":"Crucible e5b3755f-5ec3-4c87-a0e0-2f3ce464924e has session id: 8f15ac6a-2712-41bb-83de-7afe405b861c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.548204754Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189032023-09-22T23:32:47.629ZCrucible runtime is spawned
189042023-09-22T23:32:47.632Z{"msg":The guest has requested activation with gen:144
189052023-09-22T23:32:47.635Z"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.548441564Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753,"task":"up_listen"}
189062023-09-22T23:32:47.637Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.548513938Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189072023-09-22T23:32:47.641Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.548559062Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189082023-09-22T23:32:47.644Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.548598583Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753,"looper":"0"}
189092023-09-22T23:32:47.647Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.548935632Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753,"looper":"1"}
189102023-09-22T23:32:47.649Z{"msg":"[0] e5b3755f-5ec3-4c87-a0e0-2f3ce464924e looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.549002323Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753,"looper":"0"}
189112023-09-22T23:32:47.652Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.549047066Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753,"looper":"2"}
189122023-09-22T23:32:47.655Z{"msg":"[2] e5b3755f-5ec3-4c87-a0e0-2f3ce464924e looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.549251899Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753,"looper":"2"}
189132023-09-22T23:32:47.658Z{"msg":"e5b3755f-5ec3-4c87-a0e0-2f3ce464924e active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.549542035Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189142023-09-22T23:32:47.660Z{"msg":"Set desired generation to :144","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.54959476Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189152023-09-22T23:32:47.663Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.549652205Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189162023-09-22T23:32:47.666Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.550009508Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189172023-09-22T23:32:47.668Z{"msg":"[0] e5b3755f-5ec3-4c87-a0e0-2f3ce464924e (76b097fe-9b01-4a7a-820c-7fbe89f30f14) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.550406153Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189182023-09-22T23:32:47.671Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.550486156Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189192023-09-22T23:32:47.674Z{"msg":"[0] client is_active_req TRUE, promote! session 76b097fe-9b01-4a7a-820c-7fbe89f30f14","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.550563992Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189202023-09-22T23:32:47.677Z{"msg":"[2] e5b3755f-5ec3-4c87-a0e0-2f3ce464924e (76b097fe-9b01-4a7a-820c-7fbe89f30f14) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.550620911Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189212023-09-22T23:32:47.680Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.550690708Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189222023-09-22T23:32:47.682Z{"msg":"[2] client is_active_req TRUE, promote! session 76b097fe-9b01-4a7a-820c-7fbe89f30f14","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.550737161Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189232023-09-22T23:32:47.685Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.551059075Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189242023-09-22T23:32:47.687Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.551139065Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189252023-09-22T23:32:47.690Z{"msg":"e5b3755f-5ec3-4c87-a0e0-2f3ce464924e WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.551215277Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189262023-09-22T23:32:47.693Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.551296683Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189272023-09-22T23:32:47.696Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.551377333Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189282023-09-22T23:32:47.698Z{"msg":"e5b3755f-5ec3-4c87-a0e0-2f3ce464924e WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.551450973Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189292023-09-22T23:32:47.701Z{"msg":"[0] e5b3755f-5ec3-4c87-a0e0-2f3ce464924e (76b097fe-9b01-4a7a-820c-7fbe89f30f14) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.553523685Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189302023-09-22T23:32:47.704Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.553596628Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189312023-09-22T23:32:47.707Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:47.553619712Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189322023-09-22T23:32:47.710Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.553738021Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189332023-09-22T23:32:47.713Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.553789568Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189342023-09-22T23:32:47.716Z{"msg":"e5b3755f-5ec3-4c87-a0e0-2f3ce464924e WaitQuorum New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.553832379Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189352023-09-22T23:32:47.719Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.553871949Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189362023-09-22T23:32:47.721Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.553910165Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189372023-09-22T23:32:47.724Z{"msg":"[2] e5b3755f-5ec3-4c87-a0e0-2f3ce464924e (76b097fe-9b01-4a7a-820c-7fbe89f30f14) WaitQuorum New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.554702369Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189382023-09-22T23:32:47.727Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.554740727Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189392023-09-22T23:32:47.730Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:47.554759682Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189402023-09-22T23:32:47.733Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.55477909Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189412023-09-22T23:32:47.736Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.554820217Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189422023-09-22T23:32:47.739Z{"msg":"e5b3755f-5ec3-4c87-a0e0-2f3ce464924e WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.55484146Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189432023-09-22T23:32:47.742Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.554879653Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189442023-09-22T23:32:47.745Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:47.554899605Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189452023-09-22T23:32:48.551Z{"msg":"[1] e5b3755f-5ec3-4c87-a0e0-2f3ce464924e looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.549943463Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753,"looper":"1"}
189462023-09-22T23:32:48.554Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.550032114Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189472023-09-22T23:32:48.557Z{"msg":"[1] e5b3755f-5ec3-4c87-a0e0-2f3ce464924e (76b097fe-9b01-4a7a-820c-7fbe89f30f14) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.551243441Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189482023-09-22T23:32:48.560Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.551278801Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189492023-09-22T23:32:48.563Z{"msg":"[1] client is_active_req TRUE, promote! session 76b097fe-9b01-4a7a-820c-7fbe89f30f14","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.551300288Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189502023-09-22T23:32:48.567Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.551974365Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189512023-09-22T23:32:48.571Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.55203658Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189522023-09-22T23:32:48.574Z{"msg":"e5b3755f-5ec3-4c87-a0e0-2f3ce464924e WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.552081152Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189532023-09-22T23:32:48.576Z{"msg":"[1] e5b3755f-5ec3-4c87-a0e0-2f3ce464924e (76b097fe-9b01-4a7a-820c-7fbe89f30f14) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.55441941Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189542023-09-22T23:32:48.579Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.554476927Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189552023-09-22T23:32:48.582Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:48.554500081Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189562023-09-22T23:32:48.585Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.554518215Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189572023-09-22T23:32:48.589Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.55455718Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189582023-09-22T23:32:48.592Z{"msg":"e5b3755f-5ec3-4c87-a0e0-2f3ce464924e WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.554597888Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189592023-09-22T23:32:48.600Z{"msg":"[0]R flush_numbers[0..12]: [222, 202, 217, 219, 219, 222, 220, 215, 212, 215, 220, 220]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.55463864Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189602023-09-22T23:32:48.604Z{"msg":"[0]R generation[0..12]: [143, 132, 141, 142, 142, 143, 142, 140, 140, 140, 142, 142]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.554679177Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189612023-09-22T23:32:48.606Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.554720035Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189622023-09-22T23:32:48.610Z{"msg":"[1]R flush_numbers[0..12]: [222, 202, 217, 219, 219, 222, 220, 215, 212, 215, 220, 220]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.554760447Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189632023-09-22T23:32:48.613Z{"msg":"The guest has finished waiting for activation with:144
189642023-09-22T23:32:48.616ZWait for a query_work_queue command to finish before sending IO
189652023-09-22T23:32:48.619Z[1]R generation[0..12]: [143, 132, 141, 142, 142, 143, 142, 140, 140, 140, 142, 142]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.555045565Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189662023-09-22T23:32:48.622Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.555114158Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189672023-09-22T23:32:48.625Z{"msg":"[2]R flush_numbers[0..12]: [222, 202, 217, 219, 219, 222, 220, 215, 212, 215, 220, 220]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.555159315Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189682023-09-22T23:32:48.627Z{"msg":"[2]R generation[0..12]: [143, 132, 141, 142, 142, 143, 142, 140, 140, 140, 142, 142]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.55520864Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189692023-09-22T23:32:48.630Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.555257919Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189702023-09-22T23:32:48.633Z{"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.555298528Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189712023-09-22T23:32:48.636Z{"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.555340423Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189722023-09-22T23:32:48.639Z{"msg":"Next flush: 223","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.555388252Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189732023-09-22T23:32:48.642Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.555427889Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189742023-09-22T23:32:48.645Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.555474493Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189752023-09-22T23:32:48.648Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.555525192Z","hostname":"ip-10-150-1-177.us-west-2.compute.internalRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
189762023-09-22T23:32:48.651Z","pid":1753}
189772023-09-22T23:32:48.654Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.555637372Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189782023-09-22T23:32:48.657Z{"msg":"e5b3755f-5ec3-4c87-a0e0-2f3ce464924e is now active with session: 76b097fe-9b01-4a7a-820c-7fbe89f30f14","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.555685516Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189792023-09-22T23:32:48.660Z{"msg":"e5b3755f-5ec3-4c87-a0e0-2f3ce464924e Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.555725729Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189802023-09-22T23:32:48.662Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.555765156Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189812023-09-22T23:32:48.665Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.555803658Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189822023-09-22T23:32:48.668Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.555841689Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189832023-09-22T23:32:48.671Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.555880676Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189842023-09-22T23:32:48.674Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.555918246Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189852023-09-22T23:32:48.677Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.555956192Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189862023-09-22T23:32:48.680Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.555994414Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189872023-09-22T23:32:48.683Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.556032074Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189882023-09-22T23:32:48.688Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.556070052Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189892023-09-22T23:32:48.691Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.556108303Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189902023-09-22T23:32:48.694Z{"msg":"[1] Starts cmd_loopLoading write count information from file "/tmp/test_repair_verify.data"
189912023-09-22T23:32:48.696Z","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.556153355Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1753}
189922023-09-22T23:32:48.699ZRun Repair workload
189932023-09-22T23:32:48.702Z01/30 Write block 17 len 12288 data: 9 9 8
189942023-09-22T23:32:48.704Z02/30 Write block 149 len 32768 data: 12 12 12 11 8 10 11 10
189952023-09-22T23:32:48.707Z03/30 Write block 306 len 8192 data: 10 11
189962023-09-22T23:32:48.710Z04/30 Read block 154 len 8192
189972023-09-22T23:32:48.713Z05/30 Flush
189982023-09-22T23:32:48.715Z06/30 Write block 59 len 12288 data: 11 15 15
189992023-09-22T23:32:48.718Z07/30 Read block 523 len 32768
190002023-09-22T23:32:48.721Z08/30 Write block 381 len 8192 data: 10 9
190012023-09-22T23:32:48.724Z09/30 Write block 426 len 36864 data: 10 12 13 13 16 14 16 15 13
190022023-09-22T23:32:48.726Z10/30 Read block 114 len 36864
190032023-09-22T23:32:48.729Z11/30 Read block 71 len 4096
190042023-09-22T23:32:48.732Z12/30 Write block 241 len 32768 data: 15 13 9 8 9 6 7 7
190052023-09-22T23:32:48.735Z13/30 Read block 569 len 36864
190062023-09-22T23:32:48.738Z14/30 Read block 191 len 24576
190072023-09-22T23:32:48.741Z15/30 Flush
190082023-09-22T23:32:48.744Z16/30 Write block 486 len 36864 data: 17 14 18 21 18 19 16 16 14
190092023-09-22T23:32:48.747Z17/30 Read block 222 len 8192
190102023-09-22T23:32:48.750Z18/30 Read block 426 len 4096
190112023-09-22T23:32:48.753Z19/30 Write block 365 len 20480 data: 7 8 8 10 9
190122023-09-22T23:32:48.756Z20/30 Write block 458 len 24576 data: 12 15 14 13 11 12
190132023-09-22T23:32:48.758Z21/30 Write block 15 len 20480 data: 11 10 10 10 9
190142023-09-22T23:32:48.761Z22/30 Flush
190152023-09-22T23:32:48.764Z23/30 Write block 443 len 4096 data: 16
190162023-09-22T23:32:48.767Z24/30 Read block 234 len 36864
190172023-09-22T23:32:48.770Z25/30 Read block 165 len 16384
190182023-09-22T23:32:48.773Z26/30 Read block 518 len 8192
190192023-09-22T23:32:48.776Z27/30 Read block 302 len 4096
190202023-09-22T23:32:48.779Z28/30 Write block 585 len 4096 data: 8
190212023-09-22T23:32:48.781Z29/30 Read block 8 len 40960
190222023-09-22T23:32:48.784Z30/30 Write block 481 len 32768 data: 7 10 9 14 14 18 15 19
190232023-09-22T23:32:48.787Z----------------------------------------------------------------
190242023-09-22T23:32:48.790Z Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30
190252023-09-22T23:32:49.796ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
190262023-09-22T23:32:49.799Z 1 Acked 1000 Write 3 Done Done Done false
190272023-09-22T23:32:49.802Z 2 Acked 1001 Write 8 Done Done Done false
190282023-09-22T23:32:49.805Z 3 Acked 1002 Write 2 Done Done Done false
190292023-09-22T23:32:49.808Z 4 Acked 1003 Read 2 Done Done Done false
190302023-09-22T23:32:49.811Z 5 Acked 1004 Flush 0 Done Sent Done false
190312023-09-22T23:32:49.814Z 6 Acked 1005 Write 3 Done Sent Done false
190322023-09-22T23:32:49.817Z 7 Acked 1006 Read 8 Done Sent Done false
190332023-09-22T23:32:49.819Z 8 Acked 1007 Write 2 Done Sent Done false
190342023-09-22T23:32:49.822Z 9 Acked 1008 Write 9 Done Sent Done false
190352023-09-22T23:32:49.825Z 10 Acked 1009 Read 9 Done Sent Done false
190362023-09-22T23:32:49.828Z 11 Acked 1010 Read 1 Done Sent Done false
190372023-09-22T23:32:49.831Z 12 Acked 1011 Write 8 Done Sent Done false
190382023-09-22T23:32:49.833Z 13 Acked 1012 Read 9 Done Sent Done false
190392023-09-22T23:32:49.836Z 14 Acked 1013 Read 6 Done Sent Done false
190402023-09-22T23:32:49.839Z 15 Acked 1014 Flush 0 Done Sent Done false
190412023-09-22T23:32:49.842Z 16 Acked 1015 Write 9 Done Sent Done false
190422023-09-22T23:32:49.845Z 17 Acked 1016 Read 2 Done Sent Done false
190432023-09-22T23:32:49.848Z 18 Acked 1017 Read 1 Done Sent Done false
190442023-09-22T23:32:49.851Z 19 Acked 1018 Write 5 Done Sent Done false
190452023-09-22T23:32:49.853Z 20 Acked 1019 Write 6 Done Sent Done false
190462023-09-22T23:32:49.856Z 21 Acked 1020 Write 5 Done Sent Done false
190472023-09-22T23:32:49.859Z 22 Acked 1021 Flush 0 Done Sent Done false
190482023-09-22T23:32:49.862Z 23 Acked 1022 Write 1 Done Sent Done false
190492023-09-22T23:32:49.865Z 24 Acked 1023 Read 9 Done Sent Done false
190502023-09-22T23:32:49.867Z 25 Acked 1024 Read 4 Done Sent Done false
190512023-09-22T23:32:49.870Z 26 Acked 1025 Read 2 Done Sent Done false
190522023-09-22T23:32:49.873Z 27 Acked 1026 Read 1 Done Sent Done false
190532023-09-22T23:32:49.876Z 28 Acked 1027 Write 1 Done Sent Done false
190542023-09-22T23:32:49.878Z 29 Acked 1028 Read 10 Done Sent Sent false
190552023-09-22T23:32:49.881Z 30 Acked 1029 Write 8 Sent Sent Sent false
190562023-09-22T23:32:49.884Z STATES DS:0 DS:1 DS:2 TOTAL
190572023-09-22T23:32:49.887Z New 0 0 0 0
190582023-09-22T23:32:49.890Z Sent 1 26 2 29
190592023-09-22T23:32:49.893Z Done 29 4 28 61
190602023-09-22T23:32:49.896Z Skipped 0 0 0 0
190612023-09-22T23:32:49.899Z Error 0 0 0 0
190622023-09-22T23:32:49.902ZLast Flush: 1021 0 1021
190632023-09-22T23:32:49.905ZDownstairs last five completed:
190642023-09-22T23:32:49.908ZUpstairs last five completed: 30 29 28 27 26 25
190652023-09-22T23:32:49.910ZWrote out file "/tmp/test_repair_verify.data"
190662023-09-22T23:32:49.913Z
190672023-09-22T23:32:49.916ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
190682023-09-22T23:32:49.919Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.6917745Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1754}
190692023-09-22T23:32:49.922Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.692033184Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1754}
190702023-09-22T23:32:49.925Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.692055645Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1754}
190712023-09-22T23:32:49.927Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.716292916Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1754}
190722023-09-22T23:32:49.930Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.716326623Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1754}
190732023-09-22T23:32:49.933Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.716345709Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1754}
190742023-09-22T23:32:49.936Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.744329907Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1754}
190752023-09-22T23:32:49.938Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.744364217Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1754}
190762023-09-22T23:32:49.941Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:48.744383793Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1754}
190772023-09-22T23:32:49.944ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
190782023-09-22T23:32:49.947Z 0 000-019 144 143 144 225 222 225 F T F <---
190792023-09-22T23:32:49.949Z 1 020-039 132 132 132 202 202 202 F F F
190802023-09-22T23:32:49.952ZError: Difference in extent metadata found!
190812023-09-22T23:32:49.955Z 2 040-059 144 141 144 224 217 224 F F F <---
190822023-09-22T23:32:49.958Z 3 060-079 144 142 144 224 219 224 F F F <---
190832023-09-22T23:32:49.961Z 4 080-099 142 142 142 219 219 219 F F F
190842023-09-22T23:32:49.964Z 5 100-119 143 143 143 222 222 222 F F F
190852023-09-22T23:32:49.967Z 6 120-139 142 142 142 220 220 220 F F F
190862023-09-22T23:32:49.970Z 7 140-159 144 140 144 223 215 223 F T F <---
190872023-09-22T23:32:49.973Z 8 160-179 140 140 140 212 212 212 F F F
190882023-09-22T23:32:49.976Z 9 180-199 140 140 140 215 215 215 F F F
190892023-09-22T23:32:49.979Z 10 200-219 142 142 142 220 220 220 F F F
190902023-09-22T23:32:49.982Z 11 220-239 142 142 142 220 220 220 F F F
190912023-09-22T23:32:49.985Z 12 240-259 144 142 144 224 220 224 F F F <---
190922023-09-22T23:32:49.988Z 13 260-279 139 139 139 211 211 211 F F F
190932023-09-22T23:32:49.991Z 14 280-299 140 140 140 216 216 216 F F F
190942023-09-22T23:32:49.994Z 15 300-319 144 141 144 223 217 223 F T F <---
190952023-09-22T23:32:49.996Z 16 320-339 142 142 142 218 218 218 F F F
190962023-09-22T23:32:49.999Z 17 340-359 140 140 140 215 215 215 F F F
190972023-09-22T23:32:50.002Z 18 360-379 144 143 144 225 222 225 F F F <---
190982023-09-22T23:32:50.005Z 19 380-399 144 135 144 224 205 224 F F F <---
190992023-09-22T23:32:50.007Z 20 400-419 142 142 142 219 219 219 F F F
191002023-09-22T23:32:50.010Z 21 420-439 144 142 144 224 219 224 F F F <---
191012023-09-22T23:32:50.013Z 22 440-459 144 140 144 225 216 225 T F T <---
191022023-09-22T23:32:50.016Z 23 460-479 144 140 144 225 215 225 F F F <---
191032023-09-22T23:32:50.018Z 24 480-499 144 137 144 225 209 225 T F T <---
191042023-09-22T23:32:50.021Z 25 500-519 143 143 143 222 222 222 F F F
191052023-09-22T23:32:50.024Z 26 520-539 139 139 139 211 211 211 F F F
191062023-09-22T23:32:50.026Z 27 540-559 136 136 136 208 208 208 F F F
191072023-09-22T23:32:50.029Z 28 560-579 143 143 143 222 222 222 F F F
191082023-09-22T23:32:50.033Z 29 580-599 143 143 143 222 222 222 T F T <---
191092023-09-22T23:32:50.036ZMax gen: 144, Max flush: 225
191102023-09-22T23:32:50.039ZA Difference in extent metadata is expected here
191112023-09-22T23:32:50.043ZOn loop 72
191122023-09-22T23:32:50.046Z
191132023-09-22T23:32:50.049ZVerifying data now
191142023-09-22T23:32:55.165ZNo change after verify
191152023-09-22T23:32:55.168ZLoop: 72 Downstairs dump after verify (and repair):
191162023-09-22T23:32:55.184Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.182743257Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1759}
191172023-09-22T23:32:55.187Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.183106124Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1759}
191182023-09-22T23:32:55.189Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.183131446Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1759}
191192023-09-22T23:32:55.203Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.202261544Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1759}
191202023-09-22T23:32:55.206Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.202316856Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1759}
191212023-09-22T23:32:55.209Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.202337402Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1759}
191222023-09-22T23:32:55.214Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.213523995Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1759}
191232023-09-22T23:32:55.217Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.213575537Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1759}
191242023-09-22T23:32:55.220Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.213595913Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1759}
191252023-09-22T23:32:55.225ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
191262023-09-22T23:32:55.228Z 0 000-019 144 144 144 225 225 225 F F F
191272023-09-22T23:32:55.231Z 1 020-039 132 132 132 202 202 202 F F F
191282023-09-22T23:32:55.234Z 2 040-059 144 144 144 224 224 224 F F F
191292023-09-22T23:32:55.237Z 3 060-079 144 144 144 224 224 224 F F F
191302023-09-22T23:32:55.240Z 4 080-099 142 142 142 219 219 219 F F F
191312023-09-22T23:32:55.242Z 5 100-119 143 143 143 222 222 222 F F F
191322023-09-22T23:32:55.245Z 6 120-139 142 142 142 220 220 220 F F F
191332023-09-22T23:32:55.269Z 7 140-159 144 144 144 223 223 223 F F F
191342023-09-22T23:32:55.272Z 8 160-179 140 140 140 212 212 212 F F F
191352023-09-22T23:32:55.275Z 9 180-199 140 140 140 215 215 215 F F F
191362023-09-22T23:32:55.277Z 10 200-219 142 142 142 220 220 220 F F F
191372023-09-22T23:32:55.280Z 11 220-239 142 142 142 220 220 220 F F F
191382023-09-22T23:32:55.283Z 12 240-259 144 144 144 224 224 224 F F F
191392023-09-22T23:32:55.286Z 13 260-279 139 139 139 211 211 211 F F F
191402023-09-22T23:32:55.289Z 14 280-299 140 140 140 216 216 216 F F F
191412023-09-22T23:32:55.292Z 15 300-319 144 144 144 223 223 223 F F F
191422023-09-22T23:32:55.295Z 16 320-339 142 142 142 218 218 218 F F F
191432023-09-22T23:32:55.297Z 17 340-359 140 140 140 215 215 215 F F F
191442023-09-22T23:32:55.300Z 18 360-379 144 144 144 225 225 225 F F F
191452023-09-22T23:32:55.305Z 19 380-399 144 144 144 224 224 224 F F F
191462023-09-22T23:32:55.308Z 20 400-419 142 142 142 219 219 219 F F F
191472023-09-22T23:32:55.311Z 21 420-439 144 144 144 224 224 224 F F F
191482023-09-22T23:32:55.314Z 22 440-459 145 145 145 226 226 226 F F F
191492023-09-22T23:32:55.316Z 23 460-479 144 144 144 225 225 225 F F F
191502023-09-22T23:32:55.319Z 24 480-499 145 145 145 226 226 226 F F F
191512023-09-22T23:32:55.322Z 25 500-519 143 143 143 222 222 222 F F F
191522023-09-22T23:32:55.325Z 26 520-539 139 139 139 211 211 211 F F F
191532023-09-22T23:32:55.327Z 27 540-559 136 136 136 208 208 208 F F F
191542023-09-22T23:32:55.330Z 28 560-579 143 143 143 222 222 222 F F F
191552023-09-22T23:32:55.333Z 29 580-599 145 145 145 226 226 226 F F F
191562023-09-22T23:32:55.335ZMax gen: 145, Max flush: 226
191572023-09-22T23:32:55.338Z
191582023-09-22T23:32:55.341ZBegin loop 73
191592023-09-22T23:32:55.343ZDownstairs to restart: 1
191602023-09-22T23:32:55.346Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.264232525Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191612023-09-22T23:32:55.349Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.264606738Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191622023-09-22T23:32:55.352Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.26466048Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191632023-09-22T23:32:55.354Z{"msg":"Crucible stats registered with UUID: d8277f1c-c90e-40d9-b97c-b8d5d2510eb9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.264685951Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191642023-09-22T23:32:55.357Z{"msg":"Crucible d8277f1c-c90e-40d9-b97c-b8d5d2510eb9 has session id: 48158479-ad08-41d4-9b78-2685fe2a0891","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.264729595Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191652023-09-22T23:32:55.360ZCrucible runtime is spawned
191662023-09-22T23:32:55.362Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.264982883Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761,"task":"up_listen"}
191672023-09-22T23:32:55.365Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.265061111Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191682023-09-22T23:32:55.368ZThe guest has requested activation with gen:146
191692023-09-22T23:32:55.371Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.265145656Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191702023-09-22T23:32:55.374Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.265394589Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761,"looper":"1"}
191712023-09-22T23:32:55.377Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.265611971Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761,"looper":"0"}
191722023-09-22T23:32:55.380Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.265878077Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761,"looper":"2"}
191732023-09-22T23:32:55.382Z{"msg":"[0] d8277f1c-c90e-40d9-b97c-b8d5d2510eb9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.266002648Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761,"looper":"0"}
191742023-09-22T23:32:55.385Z{"msg":"[2] d8277f1c-c90e-40d9-b97c-b8d5d2510eb9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.266075057Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761,"looper":"2"}
191752023-09-22T23:32:55.388Z{"msg":"d8277f1c-c90e-40d9-b97c-b8d5d2510eb9 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.266270532Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191762023-09-22T23:32:55.391Z{"msg":"Set desired generation to :146","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.266320957Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191772023-09-22T23:32:55.394Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.266448437Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191782023-09-22T23:32:55.397Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.266750957Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191792023-09-22T23:32:55.400Z{"msg":"[0] d8277f1c-c90e-40d9-b97c-b8d5d2510eb9 (d016a1f9-1576-4491-b351-1fd0aaff2ff0) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.267088905Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191802023-09-22T23:32:55.402Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.267148403Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191812023-09-22T23:32:55.405Z{"msg":"[0] client is_active_req TRUE, promote! session d016a1f9-1576-4491-b351-1fd0aaff2ff0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.267191894Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191822023-09-22T23:32:55.408Z{"msg":"[2] d8277f1c-c90e-40d9-b97c-b8d5d2510eb9 (d016a1f9-1576-4491-b351-1fd0aaff2ff0) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.267233312Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191832023-09-22T23:32:55.411Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.26725508Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191842023-09-22T23:32:55.413Z{"msg":"[2] client is_active_req TRUE, promote! session d016a1f9-1576-4491-b351-1fd0aaff2ff0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.267293257Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191852023-09-22T23:32:55.416Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.267785337Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191862023-09-22T23:32:55.419Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.267855905Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191872023-09-22T23:32:55.422Z{"msg":"d8277f1c-c90e-40d9-b97c-b8d5d2510eb9 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.267899057Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191882023-09-22T23:32:55.424Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.268007443Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191892023-09-22T23:32:55.427Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.268061571Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191902023-09-22T23:32:55.430Z{"msg":"d8277f1c-c90e-40d9-b97c-b8d5d2510eb9 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.268085074Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191912023-09-22T23:32:55.432Z{"msg":"[2] d8277f1c-c90e-40d9-b97c-b8d5d2510eb9 (d016a1f9-1576-4491-b351-1fd0aaff2ff0) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.270326422Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191922023-09-22T23:32:55.435Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.270401387Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191932023-09-22T23:32:55.438Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:55.27044785Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191942023-09-22T23:32:55.441Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.270468979Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191952023-09-22T23:32:55.443Z{"msg":"[0] d8277f1c-c90e-40d9-b97c-b8d5d2510eb9 (d016a1f9-1576-4491-b351-1fd0aaff2ff0) WaitActive New WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.270508885Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191962023-09-22T23:32:55.446Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.270548385Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191972023-09-22T23:32:55.449Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:55.270586481Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191982023-09-22T23:32:55.455Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.270625246Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
191992023-09-22T23:32:55.458Z{"msg":"d8277f1c-c90e-40d9-b97c-b8d5d2510eb9 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.270663728Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192002023-09-22T23:32:55.461Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.270683147Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192012023-09-22T23:32:55.463Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.270700994Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192022023-09-22T23:32:55.466Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.270718395Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192032023-09-22T23:32:55.469Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.270736578Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192042023-09-22T23:32:55.473Z{"msg":"d8277f1c-c90e-40d9-b97c-b8d5d2510eb9 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.270754597Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192052023-09-22T23:32:55.476Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.270771942Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192062023-09-22T23:32:55.478Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:55.270789075Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192072023-09-22T23:32:56.269Z{"msg":"[1] d8277f1c-c90e-40d9-b97c-b8d5d2510eb9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.267399857Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761,"looper":"1"}
192082023-09-22T23:32:56.292Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.267484551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192092023-09-22T23:32:56.295Z{"msg":"[1] d8277f1c-c90e-40d9-b97c-b8d5d2510eb9 (d016a1f9-1576-4491-b351-1fd0aaff2ff0) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.268576546Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192102023-09-22T23:32:56.298Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.268634234Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192112023-09-22T23:32:56.301Z{"msg":"[1] client is_active_req TRUE, promote! session d016a1f9-1576-4491-b351-1fd0aaff2ff0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.268679508Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192122023-09-22T23:32:56.304Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.26925883Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192132023-09-22T23:32:56.307Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.269320597Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192142023-09-22T23:32:56.309Z{"msg":"d8277f1c-c90e-40d9-b97c-b8d5d2510eb9 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.269367667Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192152023-09-22T23:32:56.312Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.271527287Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192162023-09-22T23:32:56.316Z{"msg":"[1] d8277f1c-c90e-40d9-b97c-b8d5d2510eb9 (d016a1f9-1576-4491-b351-1fd0aaff2ff0) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.271816809Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192172023-09-22T23:32:56.318Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.271871382Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192182023-09-22T23:32:56.321Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:32:56.271901551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192192023-09-22T23:32:56.324Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.271958046Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192202023-09-22T23:32:56.327Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.272002015Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192212023-09-22T23:32:56.330Z{"msg":"d8277f1c-c90e-40d9-b97c-b8d5d2510eb9 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.272042349Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192222023-09-22T23:32:56.333Z{"msg":"[0]R flush_numbers[0..12]: [225, 202, 224, 224, 219, 222, 220, 223, 212, 215, 220, 220]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.272083163Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192232023-09-22T23:32:56.336Z{"msg":"[0]R generation[0..12]: [144, 132, 144, 144, 142, 143, 142, 144, 140, 140, 142, 142]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.272124074Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192242023-09-22T23:32:56.339Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.272169636Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192252023-09-22T23:32:56.342Z{"msg":"[1]R flush_numbers[0..12]: [225, 202, 224, 224, 219, 222, 220, 223, 212, 215, 220, 220]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.272209654Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192262023-09-22T23:32:56.345Z{"msg":"[1]R generation[0..12]: [144, 132, 144, 144, 142, 143, 142, 144, 140, 140, 142, 142]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:146
192272023-09-22T23:32:56.348ZWait for a query_work_queue command to finish before sending IO
192282023-09-22T23:32:56.351Z,"time":"2023-09-22T23:32:56.272232581Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192292023-09-22T23:32:56.354Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.272444054Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192302023-09-22T23:32:56.357Z{"msg":"[2]R flush_numbers[0..12]: [225, 202, 224, 224, 219, 222, 220, 223, 212, 215, 220, 220]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.272521251Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192312023-09-22T23:32:56.360Z{"msg":"[2]R generation[0..12]: [144, 132, 144, 144, 142, 143, 142, 144, 140, 140, 142, 142]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.272563285Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192322023-09-22T23:32:56.362Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.272609791Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192332023-09-22T23:32:56.365Z{"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.272658986Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192342023-09-22T23:32:56.368Z{"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.272716161Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192352023-09-22T23:32:56.371Z{"msg":"Next flush: 227","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.272779414Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192362023-09-22T23:32:56.374Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.27282238Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192372023-09-22T23:32:56.377Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.272870442Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192382023-09-22T23:32:56.380Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.272890224Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192392023-09-22T23:32:56.383Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.272943025Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192402023-09-22T23:32:56.386Z{"msg":"d8277f1c-c90e-40d9-b97c-b8d5d2510eb9 is now active with session: d016a1f9-1576-4491-b351-1fd0aaff2ff0","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.272992463Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192412023-09-22T23:32:56.389Z{"msg":"d8277f1c-c90e-40d9-b97c-b8d5d2510eb9 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.273035108Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192422023-09-22T23:32:56.391Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.273057327Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192432023-09-22T23:32:56.394Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.273110536Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192442023-09-22T23:32:56.397Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.27317649Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192452023-09-22T23:32:56.400Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.27322241Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192462023-09-22T23:32:56.402Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.273261827Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192472023-09-22T23:32:56.405Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.27330045Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192482023-09-22T23:32:56.408Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.273344498Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192492023-09-22T23:32:56.411Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
192502023-09-22T23:32:56.414Z,"time":"2023-09-22T23:32:56.273400025Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192512023-09-22T23:32:56.416Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.273505129Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192522023-09-22T23:32:56.419Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.273551704Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192532023-09-22T23:32:56.422Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.273590631Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1761}
192542023-09-22T23:32:56.425ZLoading write count information from file "/tmp/test_repair_verify.data"
192552023-09-22T23:32:56.428ZRun Repair workload
192562023-09-22T23:32:56.430Z01/30 Write block 234 len 28672 data: 9 10 11 11 11 13 15
192572023-09-22T23:32:56.433Z02/30 Read block 472 len 36864
192582023-09-22T23:32:56.436Z03/30 Read block 550 len 20480
192592023-09-22T23:32:56.439Z04/30 Write block 287 len 32768 data: 11 10 8 11 14 20 17 16
192602023-09-22T23:32:56.441Z05/30 Write block 437 len 36864 data: 14 16 14 11 14 14 17 16 15
192612023-09-22T23:32:56.444Z06/30 Read block 420 len 12288
192622023-09-22T23:32:56.447Z07/30 Write block 509 len 36864 data: 13 12 11 14 16 18 16 14 15
192632023-09-22T23:32:56.450Z08/30 Read block 314 len 4096
192642023-09-22T23:32:56.452Z09/30 Read block 221 len 12288
192652023-09-22T23:32:56.455Z10/30 Write block 420 len 20480 data: 12 12 10 10 10
192662023-09-22T23:32:56.458Z11/30 Read block 573 len 32768
192672023-09-22T23:32:56.460Z12/30 Read block 306 len 8192
192682023-09-22T23:32:56.463Z13/30 Write block 155 len 36864 data: 12 11 12 14 17 15 15 15 13
192692023-09-22T23:32:56.466Z14/30 Read block 506 len 20480
192702023-09-22T23:32:56.468Z15/30 Read block 484 len 8192
192712023-09-22T23:32:56.471Z16/30 Flush
192722023-09-22T23:32:56.474Z17/30 Write block 475 len 36864 data: 7 8 7 9 10 9 8 11 10
192732023-09-22T23:32:56.477Z18/30 Read block 570 len 8192
192742023-09-22T23:32:56.480Z19/30 Write block 247 len 12288 data: 8 8 8
192752023-09-22T23:32:56.482Z20/30 Read block 68 len 4096
192762023-09-22T23:32:56.485Z21/30 Write block 309 len 8192 data: 8 6
192772023-09-22T23:32:56.488Z22/30 Write block 186 len 36864 data: 11 11 9 10 12 12 10 10 11
192782023-09-22T23:32:56.491Z23/30 Write block 277 len 40960 data: 17 19 19 18 16 16 13 12 11 11
192792023-09-22T23:32:56.493Z24/30 Write block 102 len 16384 data: 17 19 22 24
192802023-09-22T23:32:56.496Z25/30 Write block 232 len 16384 data: 8 8 10 11
192812023-09-22T23:32:56.499Z26/30 Read block 514 len 20480
192822023-09-22T23:32:56.502Z27/30 Write block 160 len 12288 data: 16 16 16
192832023-09-22T23:32:56.505Z28/30 Write block 33 len 24576 data: 5 9 12 15 15 14
192842023-09-22T23:32:56.508Z29/30 Write block 479 len 28672 data: 11 10 9 12 11 15 15
192852023-09-22T23:32:56.510Z30/30 Write block 498 len 32768 data: 10 11 11 10 11 10 11 11
192862023-09-22T23:32:56.513Z----------------------------------------------------------------
192872023-09-22T23:32:56.516Z Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:30
192882023-09-22T23:32:56.519ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
192892023-09-22T23:32:56.522Z 1 Acked 1000 Write 7 Done Done Done false
192902023-09-22T23:32:56.525Z 2 Acked 1001 Read 9 Done Done Done false
192912023-09-22T23:32:56.528Z 3 Acked 1002 Read 5 Done Sent Done false
192922023-09-22T23:32:56.531Z 4 Acked 1003 Write 8 Done Sent Done false
192932023-09-22T23:32:56.534Z 5 Acked 1004 Write 9 Done Sent Done false
192942023-09-22T23:32:56.537Z 6 Acked 1005 Read 3 Done Sent Done false
192952023-09-22T23:32:56.540Z 7 Acked 1006 Write 9 Done Sent Done false
192962023-09-22T23:32:56.543Z 8 Acked 1007 Read 1 Done Sent Done false
192972023-09-22T23:32:56.546Z 9 Acked 1008 Read 3 Done Sent Done false
192982023-09-22T23:32:56.549Z 10 Acked 1009 Write 5 Done Sent Done false
192992023-09-22T23:32:56.551Z 11 Acked 1010 Read 8 Done Sent Done false
193002023-09-22T23:32:56.554Z 12 Acked 1011 Read 2 Done Sent Done false
193012023-09-22T23:32:56.557Z 13 Acked 1012 Write 9 Done Sent Done false
193022023-09-22T23:32:56.560Z 14 Acked 1013 Read 5 Done Sent Done false
193032023-09-22T23:32:56.563Z 15 Acked 1014 Read 2 Done Sent Done false
193042023-09-22T23:32:56.566Z 16 Acked 1015 Flush 0 Done Sent Done false
193052023-09-22T23:32:56.568Z 17 Acked 1016 Write 9 Done Sent Done false
193062023-09-22T23:32:56.571Z 18 Acked 1017 Read 2 Done Sent Done false
193072023-09-22T23:32:56.574Z 19 Acked 1018 Write 3 Done Sent Done false
193082023-09-22T23:32:56.577Z 20 Acked 1019 Read 1 Done Sent Done false
193092023-09-22T23:32:56.580Z 21 Acked 1020 Write 2 Done Sent Done false
193102023-09-22T23:32:56.583Z 22 Acked 1021 Write 9 Done Sent Done false
193112023-09-22T23:32:56.585Z 23 Acked 1022 Write 10 Done Sent Done false
193122023-09-22T23:32:56.588Z 24 Acked 1023 Write 4 Done Sent Done false
193132023-09-22T23:32:56.591Z 25 Acked 1024 Write 4 Done Sent Done false
193142023-09-22T23:32:56.594Z 26 Acked 1025 Read 5 Done Sent Done false
193152023-09-22T23:32:56.596Z 27 Acked 1026 Write 3 Done Sent Done false
193162023-09-22T23:32:56.599Z 28 Acked 1027 Write 6 Sent Sent Sent false
193172023-09-22T23:32:56.602Z 29 Acked 1028 Write 7 Sent Sent Sent false
193182023-09-22T23:32:56.605Z 30 Acked 1029 Write 8 Sent Sent Sent false
193192023-09-22T23:32:56.608Z STATES DS:0 DS:1 DS:2 TOTAL
193202023-09-22T23:32:56.610Z New 0 0 0 0
193212023-09-22T23:32:56.613Z Sent 3 28 3 34
193222023-09-22T23:32:56.616Z Done 27 2 27 56
193232023-09-22T23:32:56.619Z Skipped 0 0 0 0
193242023-09-22T23:32:56.622Z Error 0 0 0 0
193252023-09-22T23:32:56.625ZLast Flush: 1015 0 1015
193262023-09-22T23:32:56.627ZDownstairs last five completed:
193272023-09-22T23:32:56.630ZUpstairs last five completed: 30 29 28 27 26 25
193282023-09-22T23:32:56.633ZWrote out file "/tmp/test_repair_verify.data"
193292023-09-22T23:32:56.636Z
193302023-09-22T23:32:56.638ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
193312023-09-22T23:32:56.641Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.404995225Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1762}
193322023-09-22T23:32:56.644Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.40530878Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1762}
193332023-09-22T23:32:56.646Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.405333064Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1762}
193342023-09-22T23:32:56.649Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.441506912Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1762}
193352023-09-22T23:32:56.652Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.441548941Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1762}
193362023-09-22T23:32:56.655Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.441568172Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1762}
193372023-09-22T23:32:56.657Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.464657918Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1762}
193382023-09-22T23:32:56.660Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.46469019Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1762}
193392023-09-22T23:32:56.663Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:32:56.464709292Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1762}
193402023-09-22T23:32:56.666ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
193412023-09-22T23:32:56.669Z 0 000-019 144 144 144 225 225 225 F F F
193422023-09-22T23:32:56.671Z 1 020-039 132 132 132 202 202 202 T F T <---
193432023-09-22T23:32:56.674Z 2 040-059 144 144 144 224 224 224 F F F
193442023-09-22T23:32:56.677Z 3 060-079 144 144 144 224 224 224 F F F
193452023-09-22T23:32:56.679ZError: Difference in extent metadata found!
193462023-09-22T23:32:56.682Z 4 080-099 142 142 142 219 219 219 F F F
193472023-09-22T23:32:56.685Z 5 100-119 143 143 143 222 222 222 T F T <---
193482023-09-22T23:32:56.688Z 6 120-139 142 142 142 220 220 220 F F F
193492023-09-22T23:32:56.691Z 7 140-159 146 144 146 227 223 227 F F F <---
193502023-09-22T23:32:56.694Z 8 160-179 146 140 146 227 212 227 T F T <---
193512023-09-22T23:32:56.697Z 9 180-199 140 140 140 215 215 215 T F T <---
193522023-09-22T23:32:56.701Z 10 200-219 142 142 142 220 220 220 F F F
193532023-09-22T23:32:56.704Z 11 220-239 146 142 146 227 220 227 T T T <---
193542023-09-22T23:32:56.707Z 12 240-259 146 144 146 227 224 227 T T T <---
193552023-09-22T23:32:56.709Z 13 260-279 139 139 139 211 211 211 T F T <---
193562023-09-22T23:32:56.713Z 14 280-299 146 140 146 227 216 227 T F T <---
193572023-09-22T23:32:56.715Z 15 300-319 144 144 144 223 223 223 T F T <---
193582023-09-22T23:32:56.718Z 16 320-339 142 142 142 218 218 218 F F F
193592023-09-22T23:32:56.721Z 17 340-359 140 140 140 215 215 215 F F F
193602023-09-22T23:32:56.724Z 18 360-379 144 144 144 225 225 225 F F F
193612023-09-22T23:32:56.727Z 19 380-399 144 144 144 224 224 224 F F F
193622023-09-22T23:32:56.729Z 20 400-419 142 142 142 219 219 219 F F F
193632023-09-22T23:32:56.733Z 21 420-439 146 144 146 227 224 227 F F F <---
193642023-09-22T23:32:56.736Z 22 440-459 146 145 146 227 226 227 F F F <---
193652023-09-22T23:32:56.739Z 23 460-479 144 144 144 225 225 225 T F T <---
193662023-09-22T23:32:56.742Z 24 480-499 145 145 145 226 226 226 T F T <---
193672023-09-22T23:32:56.745Z 25 500-519 146 143 146 227 222 227 T F T <---
193682023-09-22T23:32:56.748Z 26 520-539 139 139 139 211 211 211 F F F
193692023-09-22T23:32:56.750Z 27 540-559 136 136 136 208 208 208 F F F
193702023-09-22T23:32:56.753Z 28 560-579 143 143 143 222 222 222 F F F
193712023-09-22T23:32:56.756Z 29 580-599 145 145 145 226 226 226 F F F
193722023-09-22T23:32:56.759ZMax gen: 146, Max flush: 227
193732023-09-22T23:32:56.762ZA Difference in extent metadata is expected here
193742023-09-22T23:32:56.765ZOn loop 73
193752023-09-22T23:32:56.768Z
193762023-09-22T23:32:56.771ZVerifying data now
193772023-09-22T23:33:03.112ZNo change after verify
193782023-09-22T23:33:03.115ZLoop: 73 Downstairs dump after verify (and repair):
193792023-09-22T23:33:03.130Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.128525851Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1767}
193802023-09-22T23:33:03.134Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.128951566Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1767}
193812023-09-22T23:33:03.138Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.129051144Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1767}
193822023-09-22T23:33:03.149Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.147587824Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1767}
193832023-09-22T23:33:03.152Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.147641162Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1767}
193842023-09-22T23:33:03.155Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.147662308Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1767}
193852023-09-22T23:33:03.161Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.158539326Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1767}
193862023-09-22T23:33:03.164Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.158591074Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1767}
193872023-09-22T23:33:03.166Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.15861169Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1767}
193882023-09-22T23:33:03.170ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
193892023-09-22T23:33:03.173Z 0 000-019 144 144 144 225 225 225 F F F
193902023-09-22T23:33:03.177Z 1 020-039 147 147 147 228 228 228 F F F
193912023-09-22T23:33:03.180Z 2 040-059 144 144 144 224 224 224 F F F
193922023-09-22T23:33:03.183Z 3 060-079 144 144 144 224 224 224 F F F
193932023-09-22T23:33:03.186Z 4 080-099 142 142 142 219 219 219 F F F
193942023-09-22T23:33:03.190Z 5 100-119 147 147 147 228 228 228 F F F
193952023-09-22T23:33:03.192Z 6 120-139 142 142 142 220 220 220 F F F
193962023-09-22T23:33:03.196Z 7 140-159 146 146 146 227 227 227 F F F
193972023-09-22T23:33:03.199Z 8 160-179 147 147 147 228 228 228 F F F
193982023-09-22T23:33:03.202Z 9 180-199 147 147 147 228 228 228 F F F
193992023-09-22T23:33:03.205Z 10 200-219 142 142 142 220 220 220 F F F
194002023-09-22T23:33:03.209Z 11 220-239 147 147 147 228 228 228 F F F
194012023-09-22T23:33:03.212Z 12 240-259 147 147 147 228 228 228 F F F
194022023-09-22T23:33:03.215Z 13 260-279 147 147 147 228 228 228 F F F
194032023-09-22T23:33:03.218Z 14 280-299 147 147 147 228 228 228 F F F
194042023-09-22T23:33:03.221Z 15 300-319 147 147 147 228 228 228 F F F
194052023-09-22T23:33:03.224Z 16 320-339 142 142 142 218 218 218 F F F
194062023-09-22T23:33:03.227Z 17 340-359 140 140 140 215 215 215 F F F
194072023-09-22T23:33:03.230Z 18 360-379 144 144 144 225 225 225 F F F
194082023-09-22T23:33:03.234Z 19 380-399 144 144 144 224 224 224 F F F
194092023-09-22T23:33:03.237Z 20 400-419 142 142 142 219 219 219 F F F
194102023-09-22T23:33:03.240Z 21 420-439 146 146 146 227 227 227 F F F
194112023-09-22T23:33:03.243Z 22 440-459 146 146 146 227 227 227 F F F
194122023-09-22T23:33:03.246Z 23 460-479 147 147 147 228 228 228 F F F
194132023-09-22T23:33:03.250Z 24 480-499 147 147 147 228 228 228 F F F
194142023-09-22T23:33:03.253Z 25 500-519 147 147 147 228 228 228 F F F
194152023-09-22T23:33:03.255Z 26 520-539 139 139 139 211 211 211 F F F
194162023-09-22T23:33:03.258Z 27 540-559 136 136 136 208 208 208 F F F
194172023-09-22T23:33:03.261Z 28 560-579 143 143 143 222 222 222 F F F
194182023-09-22T23:33:03.264Z 29 580-599 145 145 145 226 226 226 F F F
194192023-09-22T23:33:03.267ZMax gen: 147, Max flush: 228
194202023-09-22T23:33:03.271Z
194212023-09-22T23:33:03.274ZBegin loop 74
194222023-09-22T23:33:03.277ZDownstairs to restart: 0
194232023-09-22T23:33:03.280Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.20743949Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194242023-09-22T23:33:03.283Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.207812269Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194252023-09-22T23:33:03.287Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.207859541Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194262023-09-22T23:33:03.290Z{"msg":"Crucible stats registered with UUID: f2ce7ad7-2b2f-4164-90ec-8fde234b0ec4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.207881847Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194272023-09-22T23:33:03.292Z{"msg":"Crucible f2ce7ad7-2b2f-4164-90ec-8fde234b0ec4 has session id: b0f014e5-a413-4ff6-8566-51e4118e77da","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.207929045Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194282023-09-22T23:33:03.295ZCrucible runtime is spawned
194292023-09-22T23:33:03.298Z{The guest has requested activation with gen:148
194302023-09-22T23:33:03.302Z"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.20848409Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769,"task":"up_listen"}
194312023-09-22T23:33:03.305Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.208548607Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194322023-09-22T23:33:03.308Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.2085992Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769,"looper":"0"}
194332023-09-22T23:33:03.310Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.208655546Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194342023-09-22T23:33:03.313Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.208877049Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769,"looper":"2"}
194352023-09-22T23:33:03.316Z{"msg":"[2] f2ce7ad7-2b2f-4164-90ec-8fde234b0ec4 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.209136347Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769,"looper":"2"}
194362023-09-22T23:33:03.319Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.20926592Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769,"looper":"1"}
194372023-09-22T23:33:03.321Z{"msg":"[1] f2ce7ad7-2b2f-4164-90ec-8fde234b0ec4 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.209487502Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769,"looper":"1"}
194382023-09-22T23:33:03.325Z{"msg":"f2ce7ad7-2b2f-4164-90ec-8fde234b0ec4 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.209805406Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194392023-09-22T23:33:03.327Z{"msg":"Set desired generation to :148","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.209857149Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194402023-09-22T23:33:03.330Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.210050235Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194412023-09-22T23:33:03.333Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.210479878Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194422023-09-22T23:33:03.336Z{"msg":"[2] f2ce7ad7-2b2f-4164-90ec-8fde234b0ec4 (4074c1e8-2769-41b6-adab-bbabef4723b8) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.210745319Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194432023-09-22T23:33:03.339Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.210809742Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194442023-09-22T23:33:03.341Z{"msg":"[2] client is_active_req TRUE, promote! session 4074c1e8-2769-41b6-adab-bbabef4723b8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.211007793Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194452023-09-22T23:33:03.344Z{"msg":"[1] f2ce7ad7-2b2f-4164-90ec-8fde234b0ec4 (4074c1e8-2769-41b6-adab-bbabef4723b8) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.211048494Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194462023-09-22T23:33:03.347Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.211103062Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194472023-09-22T23:33:03.353Z{"msg":"[1] client is_active_req TRUE, promote! session 4074c1e8-2769-41b6-adab-bbabef4723b8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.211159672Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194482023-09-22T23:33:03.356Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.211472521Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194492023-09-22T23:33:03.359Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.211536652Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194502023-09-22T23:33:03.362Z{"msg":"f2ce7ad7-2b2f-4164-90ec-8fde234b0ec4 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.211590843Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194512023-09-22T23:33:03.365Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.211731799Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194522023-09-22T23:33:03.367Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.211830171Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194532023-09-22T23:33:03.370Z{"msg":"f2ce7ad7-2b2f-4164-90ec-8fde234b0ec4 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.21188756Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194542023-09-22T23:33:03.373Z{"msg":"[2] f2ce7ad7-2b2f-4164-90ec-8fde234b0ec4 (4074c1e8-2769-41b6-adab-bbabef4723b8) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.213820733Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194552023-09-22T23:33:03.376Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.213942731Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194562023-09-22T23:33:03.378Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:03.214060893Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194572023-09-22T23:33:03.381Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.214129852Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194582023-09-22T23:33:03.384Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.214204614Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194592023-09-22T23:33:03.388Z{"msg":"f2ce7ad7-2b2f-4164-90ec-8fde234b0ec4 New WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.214277746Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194602023-09-22T23:33:03.391Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.2143521Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194612023-09-22T23:33:03.394Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.214405153Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194622023-09-22T23:33:03.397Z{"msg":"[1] f2ce7ad7-2b2f-4164-90ec-8fde234b0ec4 (4074c1e8-2769-41b6-adab-bbabef4723b8) New WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.214449134Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194632023-09-22T23:33:03.400Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.21448887Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194642023-09-22T23:33:03.403Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:03.214528192Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194652023-09-22T23:33:03.407Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.214566451Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194662023-09-22T23:33:03.410Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.21460511Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194672023-09-22T23:33:03.413Z{"msg":"f2ce7ad7-2b2f-4164-90ec-8fde234b0ec4 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.214643837Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194682023-09-22T23:33:03.419Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.214682675Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194692023-09-22T23:33:03.425Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:03.214702295Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194702023-09-22T23:33:04.212Z{"msg":"[0] f2ce7ad7-2b2f-4164-90ec-8fde234b0ec4 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.210150653Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769,"looper":"0"}
194712023-09-22T23:33:04.216Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.21024626Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194722023-09-22T23:33:04.218Z{"msg":"[0] f2ce7ad7-2b2f-4164-90ec-8fde234b0ec4 (4074c1e8-2769-41b6-adab-bbabef4723b8) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.211449715Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194732023-09-22T23:33:04.222Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.211511602Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194742023-09-22T23:33:04.224Z{"msg":"[0] client is_active_req TRUE, promote! session 4074c1e8-2769-41b6-adab-bbabef4723b8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.211535828Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194752023-09-22T23:33:04.227Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.212160564Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194762023-09-22T23:33:04.230Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.212221108Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194772023-09-22T23:33:04.233Z{"msg":"f2ce7ad7-2b2f-4164-90ec-8fde234b0ec4 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.21226398Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194782023-09-22T23:33:04.236Z{"msg":"[0] f2ce7ad7-2b2f-4164-90ec-8fde234b0ec4 (4074c1e8-2769-41b6-adab-bbabef4723b8) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.214667082Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194792023-09-22T23:33:04.239Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.214751052Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194802023-09-22T23:33:04.242Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:04.214860684Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194812023-09-22T23:33:04.245Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.214924606Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194822023-09-22T23:33:04.248Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.214996521Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194832023-09-22T23:33:04.251Z{"msg":"f2ce7ad7-2b2f-4164-90ec-8fde234b0ec4 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.215042234Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194842023-09-22T23:33:04.254ZThe guest has finished waiting for activation with:148
194852023-09-22T23:33:04.257Z{"msg":"Wait for a query_work_queue command to finish before sending IO
194862023-09-22T23:33:04.260Z[0]R flush_numbers[0..12]: [225, 228, 224, 224, 219, 228, 220, 227, 228, 228, 220, 228]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.215086934Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194872023-09-22T23:33:04.263Z{"msg":"[0]R generation[0..12]: [144, 147, 144, 144, 142, 147, 142, 146, 147, 147, 142, 147]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.215150664Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194882023-09-22T23:33:04.265Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.215226813Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194892023-09-22T23:33:04.268Z{"msg":"[1]R flush_numbers[0..12]: [225, 228, 224, 224, 219, 228, 220, 227, 228, 228, 220, 228]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.215307177Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194902023-09-22T23:33:04.271Z{"msg":"[1]R generation[0..12]: [144, 147, 144, 144, 142, 147, 142, 146, 147, 147, 142, 147]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.215377875Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194912023-09-22T23:33:04.274Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.215460103Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194922023-09-22T23:33:04.277Z{"msg":"[2]R flush_numbers[0..12]: [225, 228, 224, 224, 219, 228, 220, 227, 228, 228, 220, 228]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.215524001Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194932023-09-22T23:33:04.280Z{"msg":"[2]R generation[0..12]: [144, 147, 144, 144, 142, 147, 142, 146, 147, 147, 142, 147]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.215577479Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194942023-09-22T23:33:04.285Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.215645334Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194952023-09-22T23:33:04.288Z{"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.215714331Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194962023-09-22T23:33:04.291Z{"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.215786001Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194972023-09-22T23:33:04.294Z{"msg":"Next flush: 229","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.215836117Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194982023-09-22T23:33:04.297Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.215875533Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
194992023-09-22T23:33:04.300Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.215913792Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
195002023-09-22T23:33:04.303Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.215952114Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
195012023-09-22T23:33:04.306Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.21599745Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
195022023-09-22T23:33:04.308Z{"msg":"f2ce7ad7-2b2f-4164-90ec-8fde234b0ec4 is now active with session: 4074c1e8-2769-41b6-adab-bbabef4723b8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.216041522Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
195032023-09-22T23:33:04.311Z{"msg":"f2ce7ad7-2b2f-4164-90ec-8fde234b0ec4 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.216080623Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
195042023-09-22T23:33:04.314Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.216127857Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
195052023-09-22T23:33:04.317Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.216170096Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
195062023-09-22T23:33:04.321Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.216207999Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
195072023-09-22T23:33:04.324Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.216256068Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
195082023-09-22T23:33:04.327Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
195092023-09-22T23:33:04.330Z,"time":"2023-09-22T23:33:04.216311138Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
195102023-09-22T23:33:04.333Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.216409396Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
195112023-09-22T23:33:04.336Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.216454166Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
195122023-09-22T23:33:04.339Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.216492818Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
195132023-09-22T23:33:04.342Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.21653053Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
195142023-09-22T23:33:04.346Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.216568382Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
195152023-09-22T23:33:04.349Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.216606246Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1769}
195162023-09-22T23:33:04.352ZLoading write count information from file "/tmp/test_repair_verify.data"
195172023-09-22T23:33:04.355ZRun Repair workload
195182023-09-22T23:33:04.358Z01/30 Write block 301 len 36864 data: 10 12 13 11 11 11 12 10 9
195192023-09-22T23:33:04.362Z02/30 Read block 454 len 16384
195202023-09-22T23:33:04.365Z03/30 Write block 377 len 12288 data: 10 11 10
195212023-09-22T23:33:04.368Z04/30 Write block 280 len 12288 data: 19 17 17
195222023-09-22T23:33:04.371Z05/30 Read block 265 len 8192
195232023-09-22T23:33:04.374Z06/30 Write block 191 len 32768 data: 13 11 11 12 12 12 10 7
195242023-09-22T23:33:04.378Z07/30 Read block 172 len 12288
195252023-09-22T23:33:04.381Z08/30 Read block 197 len 8192
195262023-09-22T23:33:04.384Z09/30 Read block 403 len 16384
195272023-09-22T23:33:04.387Z10/30 Write block 137 len 16384 data: 11 10 11 11
195282023-09-22T23:33:04.390Z11/30 Write block 345 len 36864 data: 12 13 11 11 10 10 12 12 12
195292023-09-22T23:33:04.393Z12/30 Read block 477 len 4096
195302023-09-22T23:33:04.397Z13/30 Flush
195312023-09-22T23:33:04.400Z14/30 Write block 585 len 4096 data: 9
195322023-09-22T23:33:04.403Z15/30 Read block 395 len 16384
195332023-09-22T23:33:04.406Z16/30 Write block 591 len 20480 data: 11 13 15 12 12
195342023-09-22T23:33:04.409Z17/30 Flush
195352023-09-22T23:33:04.412Z18/30 Write block 39 len 4096 data: 13
195362023-09-22T23:33:04.415Z19/30 Write block 249 len 16384 data: 9 7 6 10
195372023-09-22T23:33:04.418Z20/30 Read block 159 len 20480
195382023-09-22T23:33:04.421Z21/30 Flush
195392023-09-22T23:33:04.423Z22/30 Write block 260 len 20480 data: 10 13 14 15 14
195402023-09-22T23:33:04.426Z23/30 Write block 396 len 4096 data: 14
195412023-09-22T23:33:04.429Z24/30 Write block 439 len 28672 data: 15 12 15 15 18 17 16
195422023-09-22T23:33:04.432Z25/30 Write block 532 len 32768 data: 12 15 11 13 12 14 15 15
195432023-09-22T23:33:04.435Z26/30 Read block 156 len 20480
195442023-09-22T23:33:04.438Z27/30 Write block 480 len 36864 data: 11 10 13 12 16 16 19 16 20
195452023-09-22T23:33:04.441Z28/30 Write block 368 len 8192 data: 11 10
195462023-09-22T23:33:04.444Z29/30 Read block 475 len 20480
195472023-09-22T23:33:04.447Z30/30 Write block 301 len 24576 data: 11 13 14 12 12 12
195482023-09-22T23:33:04.470Z----------------------------------------------------------------
195492023-09-22T23:33:04.473Z Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30
195502023-09-22T23:33:04.477ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
195512023-09-22T23:33:04.480Z 1 Acked 1000 Write 9 Done Done Done false
195522023-09-22T23:33:04.483Z 2 Acked 1001 Read 4 Done Done Done false
195532023-09-22T23:33:04.486Z 3 Acked 1002 Write 3 Done Done Done false
195542023-09-22T23:33:04.488Z 4 Acked 1003 Write 3 Sent Done Done false
195552023-09-22T23:33:04.491Z 5 Acked 1004 Read 2 Sent Done Done false
195562023-09-22T23:33:04.494Z 6 Acked 1005 Write 8 Sent Done Done false
195572023-09-22T23:33:04.497Z 7 Acked 1006 Read 3 Sent Done Done false
195582023-09-22T23:33:04.500Z 8 Acked 1007 Read 2 Sent Done Done false
195592023-09-22T23:33:04.503Z 9 Acked 1008 Read 4 Sent Done Done false
195602023-09-22T23:33:04.506Z 10 Acked 1009 Write 4 Sent Done Done false
195612023-09-22T23:33:04.508Z 11 Acked 1010 Write 9 Sent Done Done false
195622023-09-22T23:33:04.511Z 12 Acked 1011 Read 1 Sent Done Done false
195632023-09-22T23:33:04.514Z 13 Acked 1012 Flush 0 Sent Done Done false
195642023-09-22T23:33:04.517Z 14 Acked 1013 Write 1 Sent Done Done false
195652023-09-22T23:33:04.520Z 15 Acked 1014 Read 4 Sent Done Done false
195662023-09-22T23:33:04.523Z 16 Acked 1015 Write 5 Sent Done Done false
195672023-09-22T23:33:04.525Z 17 Acked 1016 Flush 0 Sent Done Done false
195682023-09-22T23:33:04.529Z 18 Acked 1017 Write 1 Sent Done Done false
195692023-09-22T23:33:04.531Z 19 Acked 1018 Write 4 Sent Done Done false
195702023-09-22T23:33:04.534Z 20 Acked 1019 Read 5 Sent Done Done false
195712023-09-22T23:33:04.537Z 21 Acked 1020 Flush 0 Sent Done Done false
195722023-09-22T23:33:04.540Z 22 Acked 1021 Write 5 Sent Done Done false
195732023-09-22T23:33:04.544Z 23 Acked 1022 Write 1 Sent Done Done false
195742023-09-22T23:33:04.546Z 24 Acked 1023 Write 7 Sent Done Done false
195752023-09-22T23:33:04.549Z 25 Acked 1024 Write 8 Sent Done Done false
195762023-09-22T23:33:04.552Z 26 Acked 1025 Read 5 Sent Done Done false
195772023-09-22T23:33:04.555Z 27 Acked 1026 Write 9 Sent Done Done false
195782023-09-22T23:33:04.558Z 28 Acked 1027 Write 2 Sent Done Done false
195792023-09-22T23:33:04.560Z 29 Acked 1028 Read 5 Sent Done Done false
195802023-09-22T23:33:04.564Z 30 Acked 1029 Write 6 Sent Sent Sent false
195812023-09-22T23:33:04.567Z STATES DS:0 DS:1 DS:2 TOTAL
195822023-09-22T23:33:04.569Z New 0 0 0 0
195832023-09-22T23:33:04.572Z Sent 27 1 1 29
195842023-09-22T23:33:04.575Z Done 3 29 29 61
195852023-09-22T23:33:04.578Z Skipped 0 0 0 0
195862023-09-22T23:33:04.580Z Error 0 0 0 0
195872023-09-22T23:33:04.583ZLast Flush: 0 1020 1020
195882023-09-22T23:33:04.586ZDownstairs last five completed:
195892023-09-22T23:33:04.588ZUpstairs last five completed: 30 29 28 27 26 25
195902023-09-22T23:33:04.591ZWrote out file "/tmp/test_repair_verify.data"
195912023-09-22T23:33:04.594Z
195922023-09-22T23:33:04.597ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
195932023-09-22T23:33:04.600Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.349608134Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1770}
195942023-09-22T23:33:04.603Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.349913573Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1770}
195952023-09-22T23:33:04.606Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.34993681Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1770}
195962023-09-22T23:33:04.608Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.384061681Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1770}
195972023-09-22T23:33:04.611Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.384094164Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1770}
195982023-09-22T23:33:04.614Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.384112906Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1770}
195992023-09-22T23:33:04.616Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.40749094Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1770}
196002023-09-22T23:33:04.619Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.407523353Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1770}
196012023-09-22T23:33:04.622Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:04.407542184Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1770}
196022023-09-22T23:33:04.625ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
196032023-09-22T23:33:04.627Z 0 000-019 144 144 144 225 225 225 F F F
196042023-09-22T23:33:04.630Z 1 020-039 147 148 148 228 231 231 F F F <---
196052023-09-22T23:33:04.633Z 2 040-059 144 144 144 224 224 224 F F F
196062023-09-22T23:33:04.636Z 3 060-079 144 144 144 224 224 224 F F F
196072023-09-22T23:33:04.639ZError: Difference in extent metadata found!
196082023-09-22T23:33:04.642Z 4 080-099 142 142 142 219 219 219 F F F
196092023-09-22T23:33:04.645Z 5 100-119 147 147 147 228 228 228 F F F
196102023-09-22T23:33:04.648Z 6 120-139 142 148 148 220 229 229 F F F <---
196112023-09-22T23:33:04.650Z 7 140-159 146 148 148 227 229 229 F F F <---
196122023-09-22T23:33:04.653Z 8 160-179 147 147 147 228 228 228 F F F
196132023-09-22T23:33:04.656Z 9 180-199 147 148 148 228 229 229 F F F <---
196142023-09-22T23:33:04.659Z 10 200-219 142 142 142 220 220 220 F F F
196152023-09-22T23:33:04.662Z 11 220-239 147 147 147 228 228 228 F F F
196162023-09-22T23:33:04.665Z 12 240-259 147 148 148 228 231 231 F F F <---
196172023-09-22T23:33:04.667Z 13 260-279 147 147 147 228 228 228 F T T <---
196182023-09-22T23:33:04.670Z 14 280-299 147 148 148 228 229 229 F F F <---
196192023-09-22T23:33:04.673Z 15 300-319 147 148 148 228 229 229 T T T <---
196202023-09-22T23:33:04.676Z 16 320-339 142 142 142 218 218 218 F F F
196212023-09-22T23:33:04.679Z 17 340-359 140 148 148 215 229 229 F F F <---
196222023-09-22T23:33:04.682Z 18 360-379 144 148 148 225 229 229 T T T <---
196232023-09-22T23:33:04.685Z 19 380-399 144 144 144 224 224 224 F T T <---
196242023-09-22T23:33:04.689Z 20 400-419 142 142 142 219 219 219 F F F
196252023-09-22T23:33:04.692Z 21 420-439 146 146 146 227 227 227 F T T <---
196262023-09-22T23:33:04.694Z 22 440-459 146 146 146 227 227 227 F T T <---
196272023-09-22T23:33:04.697Z 23 460-479 147 147 147 228 228 228 F F F
196282023-09-22T23:33:04.700Z 24 480-499 147 147 147 228 228 228 F T T <---
196292023-09-22T23:33:04.703Z 25 500-519 147 147 147 228 228 228 F F F
196302023-09-22T23:33:04.706Z 26 520-539 139 139 139 211 211 211 F T T <---
196312023-09-22T23:33:04.709Z 27 540-559 136 136 136 208 208 208 F F F
196322023-09-22T23:33:04.712Z 28 560-579 143 143 143 222 222 222 F F F
196332023-09-22T23:33:04.715Z 29 580-599 145 148 148 226 230 230 F F F <---
196342023-09-22T23:33:04.719ZMax gen: 148, Max flush: 231
196352023-09-22T23:33:04.723ZA Difference in extent metadata is expected here
196362023-09-22T23:33:04.726ZOn loop 74
196372023-09-22T23:33:04.728Z
196382023-09-22T23:33:04.733ZVerifying data now
196392023-09-22T23:33:11.104ZNo change after verify
196402023-09-22T23:33:11.107ZLoop: 74 Downstairs dump after verify (and repair):
196412023-09-22T23:33:11.123Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.12137574Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1775}
196422023-09-22T23:33:11.126Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.121699429Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1775}
196432023-09-22T23:33:11.129Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.121741101Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1775}
196442023-09-22T23:33:11.141Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.140271564Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1775}
196452023-09-22T23:33:11.144Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.140328938Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1775}
196462023-09-22T23:33:11.147Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.140350868Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1775}
196472023-09-22T23:33:11.152Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.151185709Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1775}
196482023-09-22T23:33:11.155Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.151237344Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1775}
196492023-09-22T23:33:11.158Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.151257954Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1775}
196502023-09-22T23:33:11.163ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
196512023-09-22T23:33:11.166Z 0 000-019 144 144 144 225 225 225 F F F
196522023-09-22T23:33:11.169Z 1 020-039 148 148 148 231 231 231 F F F
196532023-09-22T23:33:11.172Z 2 040-059 144 144 144 224 224 224 F F F
196542023-09-22T23:33:11.175Z 3 060-079 144 144 144 224 224 224 F F F
196552023-09-22T23:33:11.177Z 4 080-099 142 142 142 219 219 219 F F F
196562023-09-22T23:33:11.180Z 5 100-119 147 147 147 228 228 228 F F F
196572023-09-22T23:33:11.183Z 6 120-139 148 148 148 229 229 229 F F F
196582023-09-22T23:33:11.185Z 7 140-159 148 148 148 229 229 229 F F F
196592023-09-22T23:33:11.188Z 8 160-179 147 147 147 228 228 228 F F F
196602023-09-22T23:33:11.191Z 9 180-199 148 148 148 229 229 229 F F F
196612023-09-22T23:33:11.194Z 10 200-219 142 142 142 220 220 220 F F F
196622023-09-22T23:33:11.197Z 11 220-239 147 147 147 228 228 228 F F F
196632023-09-22T23:33:11.200Z 12 240-259 148 148 148 231 231 231 F F F
196642023-09-22T23:33:11.203Z 13 260-279 149 149 149 232 232 232 F F F
196652023-09-22T23:33:11.206Z 14 280-299 148 148 148 229 229 229 F F F
196662023-09-22T23:33:11.209Z 15 300-319 149 149 149 232 232 232 F F F
196672023-09-22T23:33:11.212Z 16 320-339 142 142 142 218 218 218 F F F
196682023-09-22T23:33:11.215Z 17 340-359 148 148 148 229 229 229 F F F
196692023-09-22T23:33:11.217Z 18 360-379 149 149 149 232 232 232 F F F
196702023-09-22T23:33:11.221Z 19 380-399 149 149 149 232 232 232 F F F
196712023-09-22T23:33:11.224Z 20 400-419 142 142 142 219 219 219 F F F
196722023-09-22T23:33:11.226Z 21 420-439 149 149 149 232 232 232 F F F
196732023-09-22T23:33:11.229Z 22 440-459 149 149 149 232 232 232 F F F
196742023-09-22T23:33:11.232Z 23 460-479 147 147 147 228 228 228 F F F
196752023-09-22T23:33:11.235Z 24 480-499 149 149 149 232 232 232 F F F
196762023-09-22T23:33:11.238Z 25 500-519 147 147 147 228 228 228 F F F
196772023-09-22T23:33:11.241Z 26 520-539 149 149 149 232 232 232 F F F
196782023-09-22T23:33:11.244Z 27 540-559 136 136 136 208 208 208 F F F
196792023-09-22T23:33:11.247Z 28 560-579 143 143 143 222 222 222 F F F
196802023-09-22T23:33:11.250Z 29 580-599 148 148 148 230 230 230 F F F
196812023-09-22T23:33:11.253ZMax gen: 149, Max flush: 232
196822023-09-22T23:33:11.256Z
196832023-09-22T23:33:11.259ZBegin loop 75
196842023-09-22T23:33:11.262ZDownstairs to restart: 0
196852023-09-22T23:33:11.264Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.199307233Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
196862023-09-22T23:33:11.267Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.199672805Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
196872023-09-22T23:33:11.270Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.199723717Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
196882023-09-22T23:33:11.273Z{"msg":"Crucible stats registered with UUID: bf378f5d-2cc7-4578-8f0d-8f304f9fb686","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.199754939Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
196892023-09-22T23:33:11.276Z{"msg":"Crucible bf378f5d-2cc7-4578-8f0d-8f304f9fb686 has session id: df07c9af-2d49-4815-a3b8-eb331271bab8","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.199847219Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
196902023-09-22T23:33:11.279ZCrucible runtime is spawned
196912023-09-22T23:33:11.281Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30The guest has requested activation with gen:150
196922023-09-22T23:33:11.284Z,"time":"2023-09-22T23:33:11.200105114Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777,"task":"up_listen"}
196932023-09-22T23:33:11.287Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.200246685Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
196942023-09-22T23:33:11.290Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.200313757Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
196952023-09-22T23:33:11.293Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.200376992Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777,"looper":"0"}
196962023-09-22T23:33:11.296Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.200543689Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777,"looper":"2"}
196972023-09-22T23:33:11.298Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.200705534Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777,"looper":"1"}
196982023-09-22T23:33:11.301Z{"msg":"[2] bf378f5d-2cc7-4578-8f0d-8f304f9fb686 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.200751732Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777,"looper":"2"}
196992023-09-22T23:33:11.304Z{"msg":"[1] bf378f5d-2cc7-4578-8f0d-8f304f9fb686 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.200820267Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777,"looper":"1"}
197002023-09-22T23:33:11.307Z{"msg":"bf378f5d-2cc7-4578-8f0d-8f304f9fb686 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.201128653Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197012023-09-22T23:33:11.310Z{"msg":"Set desired generation to :150","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.201184784Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197022023-09-22T23:33:11.312Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.201278082Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197032023-09-22T23:33:11.316Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.201593002Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197042023-09-22T23:33:11.318Z{"msg":"[2] bf378f5d-2cc7-4578-8f0d-8f304f9fb686 (8633d31d-7043-49c2-a9e3-a1d7cd77d4d7) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.202005361Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197052023-09-22T23:33:11.321Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.202112679Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197062023-09-22T23:33:11.324Z{"msg":"[2] client is_active_req TRUE, promote! session 8633d31d-7043-49c2-a9e3-a1d7cd77d4d7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.20219158Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197072023-09-22T23:33:11.327Z{"msg":"[1] bf378f5d-2cc7-4578-8f0d-8f304f9fb686 (8633d31d-7043-49c2-a9e3-a1d7cd77d4d7) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.202268534Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197082023-09-22T23:33:11.330Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.202349859Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197092023-09-22T23:33:11.333Z{"msg":"[1] client is_active_req TRUE, promote! session 8633d31d-7043-49c2-a9e3-a1d7cd77d4d7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.202460038Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197102023-09-22T23:33:11.336Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.202740303Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197112023-09-22T23:33:11.338Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.202801374Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197122023-09-22T23:33:11.341Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.202825486Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197132023-09-22T23:33:11.344Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.202848806Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197142023-09-22T23:33:11.347Z{"msg":"bf378f5d-2cc7-4578-8f0d-8f304f9fb686 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.202887585Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197152023-09-22T23:33:11.350Z{"msg":"bf378f5d-2cc7-4578-8f0d-8f304f9fb686 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.203018018Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197162023-09-22T23:33:11.353Z{"msg":"[2] bf378f5d-2cc7-4578-8f0d-8f304f9fb686 (8633d31d-7043-49c2-a9e3-a1d7cd77d4d7) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.205063277Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197172023-09-22T23:33:11.356Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.205155889Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197182023-09-22T23:33:11.359Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:11.20518404Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197192023-09-22T23:33:11.362Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.205308643Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197202023-09-22T23:33:11.365Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.205357376Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197212023-09-22T23:33:11.368Z{"msg":"bf378f5d-2cc7-4578-8f0d-8f304f9fb686 New WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.205379641Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197222023-09-22T23:33:11.371Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.205417323Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197232023-09-22T23:33:11.374Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.205463923Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197242023-09-22T23:33:11.377Z{"msg":"[1] bf378f5d-2cc7-4578-8f0d-8f304f9fb686 (8633d31d-7043-49c2-a9e3-a1d7cd77d4d7) New WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.205513489Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197252023-09-22T23:33:11.380Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.205555701Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197262023-09-22T23:33:11.383Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:11.205576275Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197272023-09-22T23:33:11.386Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.205612534Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197282023-09-22T23:33:11.389Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.205651439Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197292023-09-22T23:33:11.392Z{"msg":"bf378f5d-2cc7-4578-8f0d-8f304f9fb686 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.205671859Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197302023-09-22T23:33:11.395Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.205708535Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197312023-09-22T23:33:11.397Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:11.205746561Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197322023-09-22T23:33:12.204Z{"msg":"[0] bf378f5d-2cc7-4578-8f0d-8f304f9fb686 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.201857728Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777,"looper":"0"}
197332023-09-22T23:33:12.207Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.201963929Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197342023-09-22T23:33:12.210Z{"msg":"[0] bf378f5d-2cc7-4578-8f0d-8f304f9fb686 (8633d31d-7043-49c2-a9e3-a1d7cd77d4d7) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.20299723Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197352023-09-22T23:33:12.213Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.203095471Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197362023-09-22T23:33:12.216Z{"msg":"[0] client is_active_req TRUE, promote! session 8633d31d-7043-49c2-a9e3-a1d7cd77d4d7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.203157859Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197372023-09-22T23:33:12.219Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.203687002Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197382023-09-22T23:33:12.224Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.203748525Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197392023-09-22T23:33:12.228Z{"msg":"bf378f5d-2cc7-4578-8f0d-8f304f9fb686 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.203793169Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197402023-09-22T23:33:12.231Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.206029246Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197412023-09-22T23:33:12.234Z{"msg":"[0] bf378f5d-2cc7-4578-8f0d-8f304f9fb686 (8633d31d-7043-49c2-a9e3-a1d7cd77d4d7) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.206261021Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197422023-09-22T23:33:12.237Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.206352729Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197432023-09-22T23:33:12.240Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:12.206406458Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197442023-09-22T23:33:12.243Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.206447209Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197452023-09-22T23:33:12.246Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.206486549Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197462023-09-22T23:33:12.248Z{"msg":"bf378f5d-2cc7-4578-8f0d-8f304f9fb686 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.206568641Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197472023-09-22T23:33:12.251Z{"msg":"[0]R flush_numbers[0..12]: [225, 231, 224, 224, 219, 228, 229, 229, 228, 229, 220, 228]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.206590641Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197482023-09-22T23:33:12.254Z{"msg":"[0]R generation[0..12]: [144, 148, 144, 144, 142, 147, 148, 148, 147, 148, 142, 147]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.206609826Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197492023-09-22T23:33:12.257Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","The guest has finished waiting for activation with:150
197502023-09-22T23:33:12.260Zv":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO
197512023-09-22T23:33:12.263Z,"time":"2023-09-22T23:33:12.206657701Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197522023-09-22T23:33:12.266Z{"msg":"[1]R flush_numbers[0..12]: [225, 231, 224, 224, 219, 228, 229, 229, 228, 229, 220, 228]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.206686838Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197532023-09-22T23:33:12.270Z{"msg":"[1]R generation[0..12]: [144, 148, 144, 144, 142, 147, 148, 148, 147, 148, 142, 147]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.206749978Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197542023-09-22T23:33:12.273Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.206788819Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197552023-09-22T23:33:12.276Z{"msg":"[2]R flush_numbers[0..12]: [225, 231, 224, 224, 219, 228, 229, 229, 228, 229, 220, 228]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.20684436Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197562023-09-22T23:33:12.279Z{"msg":"[2]R generation[0..12]: [144, 148, 144, 144, 142, 147, 148, 148, 147, 148, 142, 147]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.206888146Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197572023-09-22T23:33:12.283Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.206928809Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197582023-09-22T23:33:12.286Z{"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.206949795Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197592023-09-22T23:33:12.289Z{"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.20698741Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197602023-09-22T23:33:12.292Z{"msg":"Next flush: 233","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.207026144Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197612023-09-22T23:33:12.295Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.207071674Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197622023-09-22T23:33:12.298Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.207109473Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197632023-09-22T23:33:12.301Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.207156256Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197642023-09-22T23:33:12.304Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.207199482Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197652023-09-22T23:33:12.307Z{"msg":"bf378f5d-2cc7-4578-8f0d-8f304f9fb686 is now active with session: 8633d31d-7043-49c2-a9e3-a1d7cd77d4d7","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.207239463Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197662023-09-22T23:33:12.310Z{"msg":"bf378f5d-2cc7-4578-8f0d-8f304f9fb686 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.207278288Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197672023-09-22T23:33:12.312Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.207299645Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197682023-09-22T23:33:12.315Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.207343737Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197692023-09-22T23:33:12.318Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.207364719Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197702023-09-22T23:33:12.321Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.207401323Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197712023-09-22T23:33:12.324Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.207440859Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197722023-09-22T23:33:12.327Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.207480867Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197732023-09-22T23:33:12.330Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.207519504Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197742023-09-22T23:33:12.333Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.207557219Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197752023-09-22T23:33:12.336Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.207599282Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197762023-09-22T23:33:12.339Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
197772023-09-22T23:33:12.342Z,"time":"2023-09-22T23:33:12.207654258Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197782023-09-22T23:33:12.345Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.207758625Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1777}
197792023-09-22T23:33:12.348ZLoading write count information from file "/tmp/test_repair_verify.data"
197802023-09-22T23:33:12.351ZRun Repair workload
197812023-09-22T23:33:12.354Z01/30 Flush
197822023-09-22T23:33:12.356Z02/30 Flush
197832023-09-22T23:33:12.359Z03/30 Write block 351 len 40960 data: 13 13 13 13 13 11 10 12 15 16
197842023-09-22T23:33:12.362Z04/30 Write block 118 len 8192 data: 10 10
197852023-09-22T23:33:12.365Z05/30 Write block 387 len 12288 data: 11 11 11
197862023-09-22T23:33:12.368Z06/30 Write block 585 len 32768 data: 10 7 8 10 11 11 12 14
197872023-09-22T23:33:12.370Z07/30 Write block 338 len 16384 data: 12 11 10 8
197882023-09-22T23:33:12.373Z08/30 Read block 414 len 20480
197892023-09-22T23:33:12.376Z09/30 Read block 540 len 36864
197902023-09-22T23:33:12.379Z10/30 Write block 195 len 40960 data: 13 13 11 8 7 9 11 14 16 16
197912023-09-22T23:33:12.381Z11/30 Flush
197922023-09-22T23:33:12.384Z12/30 Write block 527 len 40960 data: 13 12 11 9 11 13 16 12 14 13
197932023-09-22T23:33:12.387Z13/30 Read block 436 len 16384
197942023-09-22T23:33:12.391Z14/30 Read block 533 len 4096
197952023-09-22T23:33:12.394Z15/30 Read block 149 len 24576
197962023-09-22T23:33:12.397Z16/30 Read block 121 len 8192
197972023-09-22T23:33:12.400Z17/30 Read block 222 len 4096
197982023-09-22T23:33:12.403Z18/30 Read block 16 len 20480
197992023-09-22T23:33:12.406Z19/30 Read block 539 len 40960
198002023-09-22T23:33:12.409Z20/30 Read block 362 len 8192
198012023-09-22T23:33:12.412Z21/30 Write block 547 len 24576 data: 10 13 16 15 15 15
198022023-09-22T23:33:12.414Z22/30 Read block 356 len 16384
198032023-09-22T23:33:12.418Z23/30 Write block 111 len 24576 data: 14 14 14 12 11 11
198042023-09-22T23:33:12.421Z24/30 Read block 37 len 8192
198052023-09-22T23:33:12.424Z25/30 Read block 533 len 32768
198062023-09-22T23:33:12.427Z26/30 Read block 131 len 12288
198072023-09-22T23:33:12.430Z27/30 Read block 348 len 8192
198082023-09-22T23:33:12.433Z28/30 Write block 138 len 28672 data: 11 12 12 11 12 11 7
198092023-09-22T23:33:12.436Z29/30 Write block 591 len 32768 data: 13 15 16 13 13 12 6 5
198102023-09-22T23:33:12.440Z30/30 Write block 331 len 36864 data: 12 12 12 16 17 17 13 13 12
198112023-09-22T23:33:12.443Z----------------------------------------------------------------
198122023-09-22T23:33:12.445Z Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:28
198132023-09-22T23:33:12.449ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
198142023-09-22T23:33:12.452Z 3 Acked 1002 Write 10 Done Done Done false
198152023-09-22T23:33:12.455Z 4 Acked 1003 Write 2 Done Done Done false
198162023-09-22T23:33:12.458Z 5 Acked 1004 Write 3 Done Done Done false
198172023-09-22T23:33:12.461Z 6 Acked 1005 Write 8 Done Done Done false
198182023-09-22T23:33:12.464Z 7 Acked 1006 Write 4 Done Done Done false
198192023-09-22T23:33:12.467Z 8 Acked 1007 Read 5 Done Done Done false
198202023-09-22T23:33:12.470Z 9 Acked 1008 Read 9 Sent Done Done false
198212023-09-22T23:33:12.473Z 10 Acked 1009 Write 10 Sent Done Done false
198222023-09-22T23:33:12.476Z 11 Acked 1010 Flush 0 Sent Done Done false
198232023-09-22T23:33:12.479Z 12 Acked 1011 Write 10 Sent Done Done false
198242023-09-22T23:33:12.481Z 13 Acked 1012 Read 4 Sent Done Done false
198252023-09-22T23:33:12.484Z 14 Acked 1013 Read 1 Sent Done Done false
198262023-09-22T23:33:12.487Z 15 Acked 1014 Read 6 Sent Done Done false
198272023-09-22T23:33:12.490Z 16 Acked 1015 Read 2 Sent Done Done false
198282023-09-22T23:33:12.493Z 17 Acked 1016 Read 1 Sent Done Done false
198292023-09-22T23:33:12.496Z 18 Acked 1017 Read 5 Sent Done Done false
198302023-09-22T23:33:12.499Z 19 Acked 1018 Read 10 Sent Done Done false
198312023-09-22T23:33:12.502Z 20 Acked 1019 Read 2 Sent Done Done false
198322023-09-22T23:33:12.505Z 21 Acked 1020 Write 6 Sent Done Done false
198332023-09-22T23:33:12.508Z 22 Acked 1021 Read 4 Sent Done Done false
198342023-09-22T23:33:12.511Z 23 Acked 1022 Write 6 Sent Done Done false
198352023-09-22T23:33:12.513Z 24 Acked 1023 Read 2 Sent Done Done false
198362023-09-22T23:33:12.516Z 25 Acked 1024 Read 8 Sent Done Done false
198372023-09-22T23:33:12.520Z 26 Acked 1025 Read 3 Sent Done Done false
198382023-09-22T23:33:12.523Z 27 Acked 1026 Read 2 Sent Done Done false
198392023-09-22T23:33:12.525Z 28 Acked 1027 Write 7 Sent Sent Sent false
198402023-09-22T23:33:12.528Z 29 Acked 1028 Write 8 Sent Sent Sent false
198412023-09-22T23:33:12.531Z 30 Acked 1029 Write 9 Sent Sent Sent false
198422023-09-22T23:33:12.534Z STATES DS:0 DS:1 DS:2 TOTAL
198432023-09-22T23:33:12.537Z New 0 0 0 0
198442023-09-22T23:33:12.540Z Sent 22 3 3 28
198452023-09-22T23:33:12.543Z Done 6 25 25 56
198462023-09-22T23:33:12.545Z Skipped 0 0 0 0
198472023-09-22T23:33:12.549Z Error 0 0 0 0
198482023-09-22T23:33:12.552ZLast Flush: 1001 1010 1010
198492023-09-22T23:33:12.555ZDownstairs last five completed: 1001 1000
198502023-09-22T23:33:12.558ZUpstairs last five completed: 30 29 28 27 26 25
198512023-09-22T23:33:12.561ZWrote out file "/tmp/test_repair_verify.data"
198522023-09-22T23:33:12.564Z
198532023-09-22T23:33:12.711ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
198542023-09-22T23:33:12.714Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.330871191Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1778}
198552023-09-22T23:33:12.717Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.331173104Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1778}
198562023-09-22T23:33:12.719Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.331196471Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1778}
198572023-09-22T23:33:12.722Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.379706104Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1778}
198582023-09-22T23:33:12.725Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.379755341Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1778}
198592023-09-22T23:33:12.728Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.379778181Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1778}
198602023-09-22T23:33:12.730Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.399629416Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1778}
198612023-09-22T23:33:12.733Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.399683312Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1778}
198622023-09-22T23:33:12.736Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:12.399722262Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1778}
198632023-09-22T23:33:12.739ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
198642023-09-22T23:33:12.742Z 0 000-019 144 144 144 225 225 225 F F F
198652023-09-22T23:33:12.745Z 1 020-039 148 148 148 231 231 231 F F F
198662023-09-22T23:33:12.748Z 2 040-059 144 144 144 224 224 224 F F F
198672023-09-22T23:33:12.751Z 3 060-079 144 144 144 224 224 224 F F F
198682023-09-22T23:33:12.754Z 4 080-099 142 142 142 219 219 219 F F F
198692023-09-22T23:33:12.757Z 5 100-119 147 150 150 228 235 235 T T T <---
198702023-09-22T23:33:12.760ZError: Difference in extent metadata found!
198712023-09-22T23:33:12.763Z 6 120-139 148 148 148 229 229 229 F T T <---
198722023-09-22T23:33:12.765Z 7 140-159 148 148 148 229 229 229 F T T <---
198732023-09-22T23:33:12.768Z 8 160-179 147 147 147 228 228 228 F F F
198742023-09-22T23:33:12.771Z 9 180-199 148 150 150 229 235 235 F F F <---
198752023-09-22T23:33:12.774Z 10 200-219 142 150 150 220 235 235 F F F <---
198762023-09-22T23:33:12.777Z 11 220-239 147 147 147 228 228 228 F F F
198772023-09-22T23:33:12.779Z 12 240-259 148 148 148 231 231 231 F F F
198782023-09-22T23:33:12.782Z 13 260-279 149 149 149 232 232 232 F F F
198792023-09-22T23:33:12.785Z 14 280-299 148 148 148 229 229 229 F F F
198802023-09-22T23:33:12.788Z 15 300-319 149 149 149 232 232 232 F F F
198812023-09-22T23:33:12.790Z 16 320-339 142 150 150 218 235 235 T F F <---
198822023-09-22T23:33:12.793Z 17 340-359 148 150 150 229 235 235 T F F <---
198832023-09-22T23:33:12.796Z 18 360-379 149 150 150 232 235 235 T F F <---
198842023-09-22T23:33:12.799Z 19 380-399 149 150 150 232 235 235 T F F <---
198852023-09-22T23:33:12.802Z 20 400-419 142 142 142 219 219 219 F F F
198862023-09-22T23:33:12.805Z 21 420-439 149 149 149 232 232 232 F F F
198872023-09-22T23:33:12.808Z 22 440-459 149 149 149 232 232 232 F F F
198882023-09-22T23:33:12.811Z 23 460-479 147 147 147 228 228 228 F F F
198892023-09-22T23:33:12.813Z 24 480-499 149 149 149 232 232 232 F F F
198902023-09-22T23:33:12.816Z 25 500-519 147 147 147 228 228 228 F F F
198912023-09-22T23:33:12.819Z 26 520-539 149 149 149 232 232 232 F T T <---
198922023-09-22T23:33:12.822Z 27 540-559 136 136 136 208 208 208 F T T <---
198932023-09-22T23:33:12.825Z 28 560-579 143 143 143 222 222 222 F F F
198942023-09-22T23:33:12.828Z 29 580-599 148 150 150 230 235 235 T T T <---
198952023-09-22T23:33:12.831ZMax gen: 150, Max flush: 235
198962023-09-22T23:33:12.834ZA Difference in extent metadata is expected here
198972023-09-22T23:33:12.837ZOn loop 75
198982023-09-22T23:33:12.840Z
198992023-09-22T23:33:12.843ZVerifying data now
199002023-09-22T23:33:18.849ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
199012023-09-22T23:33:18.852Z334,339d333
199022023-09-22T23:33:18.855Z< 11,
199032023-09-22T23:33:18.858Z< 11,
199042023-09-22T23:33:18.861Z< 11,
199052023-09-22T23:33:18.864Z< 15,
199062023-09-22T23:33:18.866Z< 16,
199072023-09-22T23:33:18.869Z< 16,
199082023-09-22T23:33:18.872Z342c336,342
199092023-09-22T23:33:18.875Z< 11,
199102023-09-22T23:33:18.878Z---
199112023-09-22T23:33:18.881Z> 12,
199122023-09-22T23:33:18.885Z> 16,
199132023-09-22T23:33:18.887Z> 17,
199142023-09-22T23:33:18.890Z> 17,
199152023-09-22T23:33:18.893Z> 13,
199162023-09-22T23:33:18.896Z> 13,
199172023-09-22T23:33:18.899Z> 12,
199182023-09-22T23:33:18.902Zdiff found after verify
199192023-09-22T23:33:18.905ZLoop: 75 Downstairs dump after verify (and repair):
199202023-09-22T23:33:18.908Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.874982375Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1786}
199212023-09-22T23:33:18.911Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.875370153Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1786}
199222023-09-22T23:33:18.914Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.875429962Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1786}
199232023-09-22T23:33:18.917Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.89480268Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1786}
199242023-09-22T23:33:18.920Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.894861304Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1786}
199252023-09-22T23:33:18.923Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.894908993Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1786}
199262023-09-22T23:33:18.926Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.90599285Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1786}
199272023-09-22T23:33:18.929Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.906045562Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1786}
199282023-09-22T23:33:18.932Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.906083743Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1786}
199292023-09-22T23:33:18.935ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
199302023-09-22T23:33:18.938Z 0 000-019 144 144 144 225 225 225 F F F
199312023-09-22T23:33:18.941Z 1 020-039 148 148 148 231 231 231 F F F
199322023-09-22T23:33:18.944Z 2 040-059 144 144 144 224 224 224 F F F
199332023-09-22T23:33:18.947Z 3 060-079 144 144 144 224 224 224 F F F
199342023-09-22T23:33:18.950Z 4 080-099 142 142 142 219 219 219 F F F
199352023-09-22T23:33:18.953Z 5 100-119 151 151 151 236 236 236 F F F
199362023-09-22T23:33:18.956Z 6 120-139 151 151 151 236 236 236 F F F
199372023-09-22T23:33:18.959Z 7 140-159 151 151 151 236 236 236 F F F
199382023-09-22T23:33:18.962Z 8 160-179 147 147 147 228 228 228 F F F
199392023-09-22T23:33:18.965Z 9 180-199 150 150 150 235 235 235 F F F
199402023-09-22T23:33:18.968Z 10 200-219 150 150 150 235 235 235 F F F
199412023-09-22T23:33:18.971Z 11 220-239 147 147 147 228 228 228 F F F
199422023-09-22T23:33:18.974Z 12 240-259 148 148 148 231 231 231 F F F
199432023-09-22T23:33:18.977Z 13 260-279 149 149 149 232 232 232 F F F
199442023-09-22T23:33:18.979Z 14 280-299 148 148 148 229 229 229 F F F
199452023-09-22T23:33:18.982Z 15 300-319 149 149 149 232 232 232 F F F
199462023-09-22T23:33:18.985Z 16 320-339 150 150 150 235 235 235 F F F
199472023-09-22T23:33:18.988Z 17 340-359 150 150 150 235 235 235 F F F
199482023-09-22T23:33:18.991Z 18 360-379 150 150 150 235 235 235 F F F
199492023-09-22T23:33:18.994Z 19 380-399 150 150 150 235 235 235 F F F
199502023-09-22T23:33:18.997Z 20 400-419 142 142 142 219 219 219 F F F
199512023-09-22T23:33:18.999Z 21 420-439 149 149 149 232 232 232 F F F
199522023-09-22T23:33:19.003Z 22 440-459 149 149 149 232 232 232 F F F
199532023-09-22T23:33:19.005Z 23 460-479 147 147 147 228 228 228 F F F
199542023-09-22T23:33:19.008Z 24 480-499 149 149 149 232 232 232 F F F
199552023-09-22T23:33:19.011Z 25 500-519 147 147 147 228 228 228 F F F
199562023-09-22T23:33:19.014Z 26 520-539 151 151 151 236 236 236 F F F
199572023-09-22T23:33:19.017Z 27 540-559 151 151 151 236 236 236 F F F
199582023-09-22T23:33:19.020Z 28 560-579 143 143 143 222 222 222 F F F
199592023-09-22T23:33:19.023Z 29 580-599 151 151 151 236 236 236 F F F
199602023-09-22T23:33:19.026ZMax gen: 151, Max flush: 236
199612023-09-22T23:33:19.029Z
199622023-09-22T23:33:19.032ZBegin loop 76
199632023-09-22T23:33:19.036ZDownstairs to restart: 1
199642023-09-22T23:33:19.039Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.956310747Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199652023-09-22T23:33:19.042Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.95671013Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199662023-09-22T23:33:19.045Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.956780641Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199672023-09-22T23:33:19.048Z{"msg":"Crucible stats registered with UUID: 4c8b7db0-96b0-4c09-81f2-23a704d0ff4c","v":Crucible runtime is spawned
199682023-09-22T23:33:19.051Z0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.956990168Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199692023-09-22T23:33:19.054Z{"msg":"Crucible 4c8b7db0-96b0-4c09-81f2-23a704d0ff4c has session id: 3cac9c9e-c9a9-41f1-bf7a-e09acb6afe9f","v":0,"name":"crucible","level":30The guest has requested activation with gen:152
199702023-09-22T23:33:19.057Z,"time":"2023-09-22T23:33:18.957053743Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199712023-09-22T23:33:19.059Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.957121317Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788,"task":"up_listen"}
199722023-09-22T23:33:19.062Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.957179415Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199732023-09-22T23:33:19.065Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.957220695Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199742023-09-22T23:33:19.068Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.95725974Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788,"looper":"0"}
199752023-09-22T23:33:19.071Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.957664541Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788,"looper":"2"}
199762023-09-22T23:33:19.074Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.957771274Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788,"looper":"1"}
199772023-09-22T23:33:19.077Z{"msg":"[0] 4c8b7db0-96b0-4c09-81f2-23a704d0ff4c looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.95787129Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788,"looper":"0"}
199782023-09-22T23:33:19.080Z{"msg":"[2] 4c8b7db0-96b0-4c09-81f2-23a704d0ff4c looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.957941243Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788,"looper":"2"}
199792023-09-22T23:33:19.083Z{"msg":"4c8b7db0-96b0-4c09-81f2-23a704d0ff4c active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.958081501Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199802023-09-22T23:33:19.086Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.95813249Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199812023-09-22T23:33:19.089Z{"msg":"Set desired generation to :152","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.958180588Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199822023-09-22T23:33:19.092Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.958473088Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199832023-09-22T23:33:19.095Z{"msg":"[2] 4c8b7db0-96b0-4c09-81f2-23a704d0ff4c (73ef32a8-c741-41fb-856b-7decc193b764) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.958794178Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199842023-09-22T23:33:19.098Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.95885689Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199852023-09-22T23:33:19.101Z{"msg":"[2] client is_active_req TRUE, promote! session 73ef32a8-c741-41fb-856b-7decc193b764","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.958880286Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199862023-09-22T23:33:19.104Z{"msg":"[0] 4c8b7db0-96b0-4c09-81f2-23a704d0ff4c (73ef32a8-c741-41fb-856b-7decc193b764) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.958939169Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199872023-09-22T23:33:19.106Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.958991168Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199882023-09-22T23:33:19.109Z{"msg":"[0] client is_active_req TRUE, promote! session 73ef32a8-c741-41fb-856b-7decc193b764","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.959039288Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199892023-09-22T23:33:19.112Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.959471788Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199902023-09-22T23:33:19.115Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.959538756Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199912023-09-22T23:33:19.118Z{"msg":"4c8b7db0-96b0-4c09-81f2-23a704d0ff4c WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.959583129Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199922023-09-22T23:33:19.121Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.959701013Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199932023-09-22T23:33:19.123Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.959756654Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199942023-09-22T23:33:19.126Z{"msg":"4c8b7db0-96b0-4c09-81f2-23a704d0ff4c WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.959780399Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199952023-09-22T23:33:19.129Z{"msg":"[2] 4c8b7db0-96b0-4c09-81f2-23a704d0ff4c (73ef32a8-c741-41fb-856b-7decc193b764) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.96174288Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199962023-09-22T23:33:19.132Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.961798567Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199972023-09-22T23:33:19.135Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:18.96182073Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199982023-09-22T23:33:19.137Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.961866913Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
199992023-09-22T23:33:19.140Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.961983089Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200002023-09-22T23:33:19.143Z{"msg":"4c8b7db0-96b0-4c09-81f2-23a704d0ff4c WaitActive New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.96204948Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200012023-09-22T23:33:19.146Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.962095793Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200022023-09-22T23:33:19.149Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.96211839Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200032023-09-22T23:33:19.152Z{"msg":"[0] 4c8b7db0-96b0-4c09-81f2-23a704d0ff4c (73ef32a8-c741-41fb-856b-7decc193b764) WaitActive New WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.962158705Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200042023-09-22T23:33:19.155Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.96219862Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200052023-09-22T23:33:19.158Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:18.962218667Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200062023-09-22T23:33:19.160Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.962236427Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200072023-09-22T23:33:19.163Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.962254384Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200082023-09-22T23:33:19.167Z{"msg":"4c8b7db0-96b0-4c09-81f2-23a704d0ff4c WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.962272838Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200092023-09-22T23:33:19.170Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.962291066Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200102023-09-22T23:33:19.172Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:18.962308384Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200112023-09-22T23:33:19.961Z{"msg":"[1] 4c8b7db0-96b0-4c09-81f2-23a704d0ff4c looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.959015551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788,"looper":"1"}
200122023-09-22T23:33:19.964Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.959097167Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200132023-09-22T23:33:19.967Z{"msg":"[1] 4c8b7db0-96b0-4c09-81f2-23a704d0ff4c (73ef32a8-c741-41fb-856b-7decc193b764) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.960378871Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200142023-09-22T23:33:19.970Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.960435751Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200152023-09-22T23:33:19.973Z{"msg":"[1] client is_active_req TRUE, promote! session 73ef32a8-c741-41fb-856b-7decc193b764","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.960479064Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200162023-09-22T23:33:19.976Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.961067829Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200172023-09-22T23:33:19.979Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.961127904Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200182023-09-22T23:33:19.982Z{"msg":"4c8b7db0-96b0-4c09-81f2-23a704d0ff4c WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.961170965Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200192023-09-22T23:33:19.985Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.963338503Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200202023-09-22T23:33:19.988Z{"msg":"[1] 4c8b7db0-96b0-4c09-81f2-23a704d0ff4c (73ef32a8-c741-41fb-856b-7decc193b764) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.963566127Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200212023-09-22T23:33:19.991Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.963620331Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200222023-09-22T23:33:19.994Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:19.963661575Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200232023-09-22T23:33:19.997Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.963700461Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200242023-09-22T23:33:20.002Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.963739155Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200252023-09-22T23:33:20.005Z{"msg":"4c8b7db0-96b0-4c09-81f2-23a704d0ff4c WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.963778289Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200262023-09-22T23:33:20.008Z{"msg":"[0]R flush_numbers[0..12]: [225, 231, 224, 224, 219, 236, 236, 236, 228, 235, 235, 228]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.963818113Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200272023-09-22T23:33:20.011Z{"msg":"[0]R generation[0..12]: [144, 148, 144, 144, 142, 151, 151, 151, 147, 150, 150, 147]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.963858222Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200282023-09-22T23:33:20.014Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.963898716Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200292023-09-22T23:33:20.017Z{"msg":"[1]R flush_numbers[0..12]: [225, 231, 224, 224, 219, 236, 236, 236, 228, 235, 235, 228]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.96393851Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200302023-09-22T23:33:20.020Z{"msg":"[1]R generation[0..12]: [144, 148, 144, 144, 142, 151, 151, 151, 147, 150, 150, 147]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.963978535Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200312023-09-22T23:33:20.023ZThe guest has finished waiting for activation with:152
200322023-09-22T23:33:20.026Z{"msg":"Wait for a query_work_queue command to finish before sending IO
200332023-09-22T23:33:20.029Z[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.964026344Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200342023-09-22T23:33:20.032Z{"msg":"[2]R flush_numbers[0..12]: [225, 231, 224, 224, 219, 236, 236, 236, 228, 235, 235, 228]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.964066027Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200352023-09-22T23:33:20.034Z{"msg":"[2]R generation[0..12]: [144, 148, 144, 144, 142, 151, 151, 151, 147, 150, 150, 147]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.964088419Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200362023-09-22T23:33:20.037Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.964140517Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200372023-09-22T23:33:20.040Z{"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.964160006Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200382023-09-22T23:33:20.043Z{"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.964187645Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200392023-09-22T23:33:20.046Z{"msg":"Next flush: 237","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.964234548Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200402023-09-22T23:33:20.049Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.964254064Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200412023-09-22T23:33:20.052Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.964290845Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200422023-09-22T23:33:20.054Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.964340733Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200432023-09-22T23:33:20.057Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.964378728Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200442023-09-22T23:33:20.060Z{"msg":"4c8b7db0-96b0-4c09-81f2-23a704d0ff4c is now active with session: 73ef32a8-c741-41fb-856b-7decc193b764","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.964418516Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200452023-09-22T23:33:20.063Z{"msg":"4c8b7db0-96b0-4c09-81f2-23a704d0ff4c Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.964457115Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200462023-09-22T23:33:20.066Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.964496971Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200472023-09-22T23:33:20.069Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.964542898Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200482023-09-22T23:33:20.072Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.964563793Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200492023-09-22T23:33:20.075Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.964599472Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200502023-09-22T23:33:20.078Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.964645767Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200512023-09-22T23:33:20.081Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.964684104Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200522023-09-22T23:33:20.084Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.964703204Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200532023-09-22T23:33:20.087Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.964738641Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200542023-09-22T23:33:20.089Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.964760231Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200552023-09-22T23:33:20.092Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.964796149Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200562023-09-22T23:33:20.095Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:19.964814974Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1788}
200572023-09-22T23:33:20.098ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
200582023-09-22T23:33:20.100ZLoading write count information from file "/tmp/test_repair_verify.data"
200592023-09-22T23:33:20.103ZRun Repair workload
200602023-09-22T23:33:20.106Z01/30 Flush
200612023-09-22T23:33:20.109Z02/30 Write block 522 len 16384 data: 11 11 10 13
200622023-09-22T23:33:20.112Z03/30 Read block 399 len 12288
200632023-09-22T23:33:20.115Z04/30 Flush
200642023-09-22T23:33:20.118Z05/30 Flush
200652023-09-22T23:33:20.120Z06/30 Write block 514 len 16384 data: 19 17 15 16
200662023-09-22T23:33:20.123Z07/30 Read block 335 len 20480
200672023-09-22T23:33:20.126Z08/30 Read block 114 len 8192
200682023-09-22T23:33:20.129Z09/30 Write block 96 len 28672 data: 13 11 14 12 13 13 18
200692023-09-22T23:33:20.132Z10/30 Read block 431 len 12288
200702023-09-22T23:33:20.135Z11/30 Read block 225 len 16384
200712023-09-22T23:33:20.138Z12/30 Read block 281 len 36864
200722023-09-22T23:33:20.141Z13/30 Flush
200732023-09-22T23:33:20.143Z14/30 Write block 215 len 32768 data: 12 11 11 10 8 10 9 8
200742023-09-22T23:33:20.148Z15/30 Flush
200752023-09-22T23:33:20.151Z16/30 Write block 456 len 28672 data: 11 12 13 16 15 14 12
200762023-09-22T23:33:20.154Z17/30 Write block 6 len 24576 data: 8 9 9 9 9 9
200772023-09-22T23:33:20.157Z18/30 Read block 163 len 28672
200782023-09-22T23:33:20.160Z19/30 Write block 79 len 36864 data: 17 15 16 13 10 9 7 8 12
200792023-09-22T23:33:20.162Z20/30 Read block 47 len 28672
200802023-09-22T23:33:20.165Z21/30 Read block 24 len 32768
200812023-09-22T23:33:20.169Z22/30 Write block 433 len 32768 data: 16 14 16 15 15 17 16 13
200822023-09-22T23:33:20.172Z23/30 Write block 207 len 8192 data: 18 21
200832023-09-22T23:33:20.175Z24/30 Read block 183 len 36864
200842023-09-22T23:33:20.178Z25/30 Read block 138 len 16384
200852023-09-22T23:33:20.181Z26/30 Read block 216 len 20480
200862023-09-22T23:33:20.184Z27/30 Read block 394 len 36864
200872023-09-22T23:33:20.187Z28/30 Write block 147 len 16384 data: 7 11 13 13
200882023-09-22T23:33:20.190Z29/30 Read block 364 len 32768
200892023-09-22T23:33:20.193Z30/30 Read block 502 len 24576
200902023-09-22T23:33:20.196Z----------------------------------------------------------------
200912023-09-22T23:33:20.198Z Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:30
200922023-09-22T23:33:20.201ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
200932023-09-22T23:33:20.205Z 1 Acked 1000 Flush 0 Done Sent Done false
200942023-09-22T23:33:20.208Z 2 Acked 1001 Write 4 Done Sent Done false
200952023-09-22T23:33:20.211Z 3 Acked 1002 Read 3 Done Sent Done false
200962023-09-22T23:33:20.214Z 4 Acked 1003 Flush 0 Done Sent Done false
200972023-09-22T23:33:20.217Z 5 Acked 1004 Flush 0 Done Sent Done false
200982023-09-22T23:33:20.220Z 6 Acked 1005 Write 4 Done Sent Done false
200992023-09-22T23:33:20.223Z 7 Acked 1006 Read 5 Done Sent Done false
201002023-09-22T23:33:20.226Z 8 Acked 1007 Read 2 Done Sent Done false
201012023-09-22T23:33:20.230Z 9 Acked 1008 Write 7 Done Sent Done false
201022023-09-22T23:33:20.233Z 10 Acked 1009 Read 3 Done Sent Done false
201032023-09-22T23:33:20.236Z 11 Acked 1010 Read 4 Done Sent Done false
201042023-09-22T23:33:20.240Z 12 Acked 1011 Read 9 Done Sent Done false
201052023-09-22T23:33:20.243Z 13 Acked 1012 Flush 0 Done Sent Done false
201062023-09-22T23:33:20.246Z 14 Acked 1013 Write 8 Done Sent Done false
201072023-09-22T23:33:20.249Z 15 Acked 1014 Flush 0 Done Sent Done false
201082023-09-22T23:33:20.252Z 16 Acked 1015 Write 7 Done Sent Done false
201092023-09-22T23:33:20.254Z 17 Acked 1016 Write 6 Done Sent Done false
201102023-09-22T23:33:20.258Z 18 Acked 1017 Read 7 Done Sent Done false
201112023-09-22T23:33:20.262Z 19 Acked 1018 Write 9 Done Sent Done false
201122023-09-22T23:33:20.273Z 20 Acked 1019 Read 7 Done Sent Done false
201132023-09-22T23:33:20.278Z 21 Acked 1020 Read 8 Done Sent Done false
201142023-09-22T23:33:20.281Z 22 Acked 1021 Write 8 Done Sent Done false
201152023-09-22T23:33:20.284Z 23 Acked 1022 Write 2 Done Sent Done false
201162023-09-22T23:33:20.287Z 24 Acked 1023 Read 9 Done Sent Done false
201172023-09-22T23:33:20.290Z 25 Acked 1024 Read 4 Done Sent Done false
201182023-09-22T23:33:20.293Z 26 Acked 1025 Read 5 Done Sent Done false
201192023-09-22T23:33:20.296Z 27 Acked 1026 Read 9 Done Sent Done false
201202023-09-22T23:33:20.299Z 28 Acked 1027 Write 4 Done Sent Done false
201212023-09-22T23:33:20.303Z 29 Acked 1028 Read 8 Done Sent Done false
201222023-09-22T23:33:20.306Z 30 Acked 1029 Read 6 Sent Sent Done false
201232023-09-22T23:33:20.309Z STATES DS:0 DS:1 DS:2 TOTAL
201242023-09-22T23:33:20.312Z New 0 0 0 0
201252023-09-22T23:33:20.315Z Sent 1 30 0 31
201262023-09-22T23:33:20.318Z Done 29 0 30 59
201272023-09-22T23:33:20.322Z Skipped 0 0 0 0
201282023-09-22T23:33:20.326Z Error 0 0 0 0
201292023-09-22T23:33:20.331ZLast Flush: 1014 0 1014
201302023-09-22T23:33:20.335ZDownstairs last five completed:
201312023-09-22T23:33:20.338ZUpstairs last five completed: 30 29 28 27 26 25
201322023-09-22T23:33:20.341ZWrote out file "/tmp/test_repair_verify.data"
201332023-09-22T23:33:20.343Z
201342023-09-22T23:33:20.346ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
201352023-09-22T23:33:20.349Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:20.103367199Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1789}
201362023-09-22T23:33:20.352Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:20.103712927Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1789}
201372023-09-22T23:33:20.355Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:20.103735743Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1789}
201382023-09-22T23:33:20.358Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:20.134037466Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1789}
201392023-09-22T23:33:20.360Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:20.134069715Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1789}
201402023-09-22T23:33:20.363Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:20.134089267Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1789}
201412023-09-22T23:33:20.366Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:20.149195537Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1789}
201422023-09-22T23:33:20.369Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:20.149226075Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1789}
201432023-09-22T23:33:20.372Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:20.149244832Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1789}
201442023-09-22T23:33:20.375ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
201452023-09-22T23:33:20.378Z 0 000-019 144 144 144 225 225 225 T F T <---
201462023-09-22T23:33:20.381Z 1 020-039 148 148 148 231 231 231 F F F
201472023-09-22T23:33:20.383Z 2 040-059 144 144 144 224 224 224 F F F
201482023-09-22T23:33:20.386ZError: Difference in extent metadata found!
201492023-09-22T23:33:20.389Z 3 060-079 144 144 144 224 224 224 T F T <---
201502023-09-22T23:33:20.392Z 4 080-099 152 142 152 240 219 240 T F T <---
201512023-09-22T23:33:20.395Z 5 100-119 152 151 152 240 236 240 F F F <---
201522023-09-22T23:33:20.398Z 6 120-139 151 151 151 236 236 236 F F F
201532023-09-22T23:33:20.401Z 7 140-159 151 151 151 236 236 236 T F T <---
201542023-09-22T23:33:20.404Z 8 160-179 147 147 147 228 228 228 F F F
201552023-09-22T23:33:20.407Z 9 180-199 150 150 150 235 235 235 F F F
201562023-09-22T23:33:20.410Z 10 200-219 152 150 152 241 235 241 T F T <---
201572023-09-22T23:33:20.413Z 11 220-239 152 147 152 241 228 241 F F F <---
201582023-09-22T23:33:20.416Z 12 240-259 148 148 148 231 231 231 F F F
201592023-09-22T23:33:20.419Z 13 260-279 149 149 149 232 232 232 F F F
201602023-09-22T23:33:20.422Z 14 280-299 148 148 148 229 229 229 F F F
201612023-09-22T23:33:20.425Z 15 300-319 149 149 149 232 232 232 F F F
201622023-09-22T23:33:20.428Z 16 320-339 150 150 150 235 235 235 F F F
201632023-09-22T23:33:20.431Z 17 340-359 150 150 150 235 235 235 F F F
201642023-09-22T23:33:20.433Z 18 360-379 150 150 150 235 235 235 F F F
201652023-09-22T23:33:20.436Z 19 380-399 150 150 150 235 235 235 F F F
201662023-09-22T23:33:20.439Z 20 400-419 142 142 142 219 219 219 F F F
201672023-09-22T23:33:20.442Z 21 420-439 149 149 149 232 232 232 T F T <---
201682023-09-22T23:33:20.445Z 22 440-459 149 149 149 232 232 232 T F T <---
201692023-09-22T23:33:20.448Z 23 460-479 147 147 147 228 228 228 T F T <---
201702023-09-22T23:33:20.450Z 24 480-499 149 149 149 232 232 232 F F F
201712023-09-22T23:33:20.454Z 25 500-519 152 147 152 240 228 240 F F F <---
201722023-09-22T23:33:20.460Z 26 520-539 152 151 152 238 236 238 F F F <---
201732023-09-22T23:33:20.463Z 27 540-559 151 151 151 236 236 236 F F F
201742023-09-22T23:33:20.466Z 28 560-579 143 143 143 222 222 222 F F F
201752023-09-22T23:33:20.469Z 29 580-599 151 151 151 236 236 236 F F F
201762023-09-22T23:33:20.472ZMax gen: 152, Max flush: 241
201772023-09-22T23:33:20.475ZA Difference in extent metadata is expected here
201782023-09-22T23:33:20.478ZOn loop 76
201792023-09-22T23:33:20.481Z
201802023-09-22T23:33:20.484ZVerifying data now
201812023-09-22T23:33:26.505ZNo change after verify
201822023-09-22T23:33:26.508ZLoop: 76 Downstairs dump after verify (and repair):
201832023-09-22T23:33:26.523Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.521281588Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1794}
201842023-09-22T23:33:26.526Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.521578293Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1794}
201852023-09-22T23:33:26.529Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.521603341Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1794}
201862023-09-22T23:33:26.541Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.539999804Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1794}
201872023-09-22T23:33:26.544Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.540052603Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1794}
201882023-09-22T23:33:26.547Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.540073413Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1794}
201892023-09-22T23:33:26.552Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.550908728Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1794}
201902023-09-22T23:33:26.555Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.550961073Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1794}
201912023-09-22T23:33:26.559Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.550981734Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1794}
201922023-09-22T23:33:26.563ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
201932023-09-22T23:33:26.567Z 0 000-019 153 153 153 242 242 242 F F F
201942023-09-22T23:33:26.570Z 1 020-039 148 148 148 231 231 231 F F F
201952023-09-22T23:33:26.573Z 2 040-059 144 144 144 224 224 224 F F F
201962023-09-22T23:33:26.576Z 3 060-079 153 153 153 242 242 242 F F F
201972023-09-22T23:33:26.580Z 4 080-099 153 153 153 242 242 242 F F F
201982023-09-22T23:33:26.582Z 5 100-119 152 152 152 240 240 240 F F F
201992023-09-22T23:33:26.586Z 6 120-139 151 151 151 236 236 236 F F F
202002023-09-22T23:33:26.589Z 7 140-159 153 153 153 242 242 242 F F F
202012023-09-22T23:33:26.592Z 8 160-179 147 147 147 228 228 228 F F F
202022023-09-22T23:33:26.595Z 9 180-199 150 150 150 235 235 235 F F F
202032023-09-22T23:33:26.598Z 10 200-219 153 153 153 242 242 242 F F F
202042023-09-22T23:33:26.600Z 11 220-239 152 152 152 241 241 241 F F F
202052023-09-22T23:33:26.603Z 12 240-259 148 148 148 231 231 231 F F F
202062023-09-22T23:33:26.606Z 13 260-279 149 149 149 232 232 232 F F F
202072023-09-22T23:33:26.609Z 14 280-299 148 148 148 229 229 229 F F F
202082023-09-22T23:33:26.612Z 15 300-319 149 149 149 232 232 232 F F F
202092023-09-22T23:33:26.615Z 16 320-339 150 150 150 235 235 235 F F F
202102023-09-22T23:33:26.618Z 17 340-359 150 150 150 235 235 235 F F F
202112023-09-22T23:33:26.621Z 18 360-379 150 150 150 235 235 235 F F F
202122023-09-22T23:33:26.624Z 19 380-399 150 150 150 235 235 235 F F F
202132023-09-22T23:33:26.627Z 20 400-419 142 142 142 219 219 219 F F F
202142023-09-22T23:33:26.630Z 21 420-439 153 153 153 242 242 242 F F F
202152023-09-22T23:33:26.633Z 22 440-459 153 153 153 242 242 242 F F F
202162023-09-22T23:33:26.636Z 23 460-479 153 153 153 242 242 242 F F F
202172023-09-22T23:33:26.639Z 24 480-499 149 149 149 232 232 232 F F F
202182023-09-22T23:33:26.642Z 25 500-519 152 152 152 240 240 240 F F F
202192023-09-22T23:33:26.645Z 26 520-539 152 152 152 238 238 238 F F F
202202023-09-22T23:33:26.647Z 27 540-559 151 151 151 236 236 236 F F F
202212023-09-22T23:33:26.650Z 28 560-579 143 143 143 222 222 222 F F F
202222023-09-22T23:33:26.653Z 29 580-599 151 151 151 236 236 236 F F F
202232023-09-22T23:33:26.657ZMax gen: 153, Max flush: 242
202242023-09-22T23:33:26.660Z
202252023-09-22T23:33:26.662ZBegin loop 77
202262023-09-22T23:33:26.665ZDownstairs to restart: 0
202272023-09-22T23:33:26.669Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.597798847Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202282023-09-22T23:33:26.672Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.59819843Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202292023-09-22T23:33:26.674Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.598258848Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202302023-09-22T23:33:26.677Z{"msg":"Crucible stats registered with UUID: 6ebb12b7-6091-439e-8bec-92c0f40567ba","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.598319137Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202312023-09-22T23:33:26.680Z{"msg":"Crucible 6ebb12b7-6091-439e-8bec-92c0f40567ba has session id: 96ee068a-ba01-47c5-a6ec-010dd8026d53","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.598376974Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202322023-09-22T23:33:26.683ZCrucible runtime is spawned
202332023-09-22T23:33:26.686ZThe guest has requested activation with gen:154
202342023-09-22T23:33:26.689Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.598642158Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796,"task":"up_listen"}
202352023-09-22T23:33:26.692Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.598725772Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202362023-09-22T23:33:26.695Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.598790892Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796,"looper":"0"}
202372023-09-22T23:33:26.698Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.598866197Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202382023-09-22T23:33:26.701Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.598979925Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796,"looper":"2"}
202392023-09-22T23:33:26.704Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.599200172Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796,"looper":"1"}
202402023-09-22T23:33:26.706Z{"msg":"[2] 6ebb12b7-6091-439e-8bec-92c0f40567ba looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.599277495Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796,"looper":"2"}
202412023-09-22T23:33:26.709Z{"msg":"[1] 6ebb12b7-6091-439e-8bec-92c0f40567ba looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.599487858Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796,"looper":"1"}
202422023-09-22T23:33:26.712Z{"msg":"6ebb12b7-6091-439e-8bec-92c0f40567ba active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.599674563Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202432023-09-22T23:33:26.715Z{"msg":"Set desired generation to :154","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.599731616Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202442023-09-22T23:33:26.718Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.599919212Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202452023-09-22T23:33:26.721Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.600342936Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202462023-09-22T23:33:26.724Z{"msg":"[2] 6ebb12b7-6091-439e-8bec-92c0f40567ba (3c4a77c1-3ae4-475a-b033-e1ceda433ef6) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.600672801Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202472023-09-22T23:33:26.726Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.600730927Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202482023-09-22T23:33:26.729Z{"msg":"[2] client is_active_req TRUE, promote! session 3c4a77c1-3ae4-475a-b033-e1ceda433ef6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.600751491Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202492023-09-22T23:33:26.732Z{"msg":"[1] 6ebb12b7-6091-439e-8bec-92c0f40567ba (3c4a77c1-3ae4-475a-b033-e1ceda433ef6) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.600866011Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202502023-09-22T23:33:26.736Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.600955978Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202512023-09-22T23:33:26.739Z{"msg":"[1] client is_active_req TRUE, promote! session 3c4a77c1-3ae4-475a-b033-e1ceda433ef6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.601009Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202522023-09-22T23:33:26.742Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.601304097Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202532023-09-22T23:33:26.745Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.601389301Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202542023-09-22T23:33:26.748Z{"msg":"6ebb12b7-6091-439e-8bec-92c0f40567ba New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.601431856Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202552023-09-22T23:33:26.750Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.601525192Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202562023-09-22T23:33:26.753Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.601560042Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202572023-09-22T23:33:26.756Z{"msg":"6ebb12b7-6091-439e-8bec-92c0f40567ba New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.601579775Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202582023-09-22T23:33:26.759Z{"msg":"[2] 6ebb12b7-6091-439e-8bec-92c0f40567ba (3c4a77c1-3ae4-475a-b033-e1ceda433ef6) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.60413949Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202592023-09-22T23:33:26.762Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.604247542Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202602023-09-22T23:33:26.765Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:26.60430205Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202612023-09-22T23:33:26.768Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.604373825Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202622023-09-22T23:33:26.771Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.604480089Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202632023-09-22T23:33:26.774Z{"msg":"6ebb12b7-6091-439e-8bec-92c0f40567ba New WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.604530384Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202642023-09-22T23:33:26.777Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.604552852Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202652023-09-22T23:33:26.780Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.604589431Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202662023-09-22T23:33:26.782Z{"msg":"[1] 6ebb12b7-6091-439e-8bec-92c0f40567ba (3c4a77c1-3ae4-475a-b033-e1ceda433ef6) New WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.604677869Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202672023-09-22T23:33:26.785Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.604725025Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202682023-09-22T23:33:26.789Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:26.604747015Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202692023-09-22T23:33:26.792Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.604783546Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202702023-09-22T23:33:26.795Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.604804109Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202712023-09-22T23:33:26.797Z{"msg":"6ebb12b7-6091-439e-8bec-92c0f40567ba New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.604840852Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202722023-09-22T23:33:26.800Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.604861012Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202732023-09-22T23:33:26.803Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:26.604878333Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202742023-09-22T23:33:27.601Z{"msg":"[0] 6ebb12b7-6091-439e-8bec-92c0f40567ba looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.599712035Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796,"looper":"0"}
202752023-09-22T23:33:27.605Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.599805977Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202762023-09-22T23:33:27.608Z{"msg":"[0] 6ebb12b7-6091-439e-8bec-92c0f40567ba (3c4a77c1-3ae4-475a-b033-e1ceda433ef6) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.600959599Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202772023-09-22T23:33:27.612Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.601014914Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202782023-09-22T23:33:27.616Z{"msg":"[0] client is_active_req TRUE, promote! session 3c4a77c1-3ae4-475a-b033-e1ceda433ef6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.601058866Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202792023-09-22T23:33:27.619Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.601746919Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202802023-09-22T23:33:27.623Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.601836188Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202812023-09-22T23:33:28.628Z{"msg":"6ebb12b7-6091-439e-8bec-92c0f40567ba WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.601861729Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202822023-09-22T23:33:28.631Z{"msg":"[0] 6ebb12b7-6091-439e-8bec-92c0f40567ba (3c4a77c1-3ae4-475a-b033-e1ceda433ef6) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.604467958Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202832023-09-22T23:33:28.634Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.604529405Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202842023-09-22T23:33:28.637Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:27.604615559Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202852023-09-22T23:33:28.640Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.604682523Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202862023-09-22T23:33:28.642Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.604749428Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202872023-09-22T23:33:28.645Z{"msg":"6ebb12b7-6091-439e-8bec-92c0f40567ba WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.604816357Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202882023-09-22T23:33:28.648Z{"msg":"[0]R flush_numbers[0..12]: [242, 231, 224, 242, 242, 240, 236, 242, 228, 235, 242, 241]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.604883027Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202892023-09-22T23:33:28.651Z{"msg":"[0]R generation[0..12]: [153, 148, 144, 153, 153, 152, 151, 153, 147, 150, 153, 152]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.604951163Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796The guest has finished waiting for activation with:154
202902023-09-22T23:33:28.654ZWait for a query_work_queue command to finish before sending IO
202912023-09-22T23:33:28.657Z}
202922023-09-22T23:33:28.660Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.605167612Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202932023-09-22T23:33:28.663Z{"msg":"[1]R flush_numbers[0..12]: [242, 231, 224, 242, 242, 240, 236, 242, 228, 235, 242, 241]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.605252792Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202942023-09-22T23:33:28.666Z{"msg":"[1]R generation[0..12]: [153, 148, 144, 153, 153, 152, 151, 153, 147, 150, 153, 152]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.605327223Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202952023-09-22T23:33:28.668Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.605358935Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202962023-09-22T23:33:28.672Z{"msg":"[2]R flush_numbers[0..12]: [242, 231, 224, 242, 242, 240, 236, 242, 228, 235, 242, 241]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.605430747Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202972023-09-22T23:33:28.674Z{"msg":"[2]R generation[0..12]: [153, 148, 144, 153, 153, 152, 151, 153, 147, 150, 153, 152]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.605479909Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202982023-09-22T23:33:28.677Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.60550378Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
202992023-09-22T23:33:28.680Z{"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.605544704Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
203002023-09-22T23:33:28.683Z{"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.605589315Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
203012023-09-22T23:33:28.686Z{"msg":"Next flush: 243","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.605636652Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
203022023-09-22T23:33:28.688Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.605685598Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
203032023-09-22T23:33:28.691Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.605726512Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
203042023-09-22T23:33:28.694Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.605769796Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
203052023-09-22T23:33:28.696Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.60580892Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
203062023-09-22T23:33:28.699Z{"msg":"6ebb12b7-6091-439e-8bec-92c0f40567ba is now active with session: 3c4a77c1-3ae4-475a-b033-e1ceda433ef6","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.605857951Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
203072023-09-22T23:33:28.702Z{"msg":"6ebb12b7-6091-439e-8bec-92c0f40567ba Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.605911948Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
203082023-09-22T23:33:28.705Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.605937095Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
203092023-09-22T23:33:28.708Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.605958143Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
203102023-09-22T23:33:28.711Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.605980372Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
203112023-09-22T23:33:28.714Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.606001679Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
203122023-09-22T23:33:28.716Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.606048322Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
203132023-09-22T23:33:28.719Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.606086808Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
203142023-09-22T23:33:28.722Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.606138429Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
203152023-09-22T23:33:28.725Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.606157665Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
203162023-09-22T23:33:28.728Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.606200065Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
203172023-09-22T23:33:28.730Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.606241089Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
203182023-09-22T23:33:28.733Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.606260333Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1796}
203192023-09-22T23:33:28.736ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
203202023-09-22T23:33:28.739ZLoading write count information from file "/tmp/test_repair_verify.data"
203212023-09-22T23:33:28.742ZRun Repair workload
203222023-09-22T23:33:28.744Z01/30 Write block 240 len 24576 data: 16 16 14 10 9 10
203232023-09-22T23:33:28.747Z02/30 Write block 44 len 16384 data: 10 11 10 11
203242023-09-22T23:33:28.750Z03/30 Flush
203252023-09-22T23:33:28.753Z04/30 Write block 90 len 40960 data: 10 13 13 13 12 14 14 12 15 13
203262023-09-22T23:33:28.756Z05/30 Read block 570 len 24576
203272023-09-22T23:33:28.759Z06/30 Write block 451 len 20480 data: 13 12 14 12 12
203282023-09-22T23:33:28.761Z07/30 Flush
203292023-09-22T23:33:28.764Z08/30 Flush
203302023-09-22T23:33:28.767Z09/30 Flush
203312023-09-22T23:33:28.770Z10/30 Write block 111 len 16384 data: 15 15 15 13
203322023-09-22T23:33:28.773Z11/30 Read block 58 len 4096
203332023-09-22T23:33:28.776Z12/30 Read block 0 len 32768
203342023-09-22T23:33:28.779Z13/30 Read block 360 len 32768
203352023-09-22T23:33:28.781Z14/30 Read block 439 len 8192
203362023-09-22T23:33:28.784Z15/30 Read block 545 len 24576
203372023-09-22T23:33:28.787Z16/30 Read block 440 len 12288
203382023-09-22T23:33:28.790Z17/30 Write block 509 len 8192 data: 14 13
203392023-09-22T23:33:28.793Z18/30 Write block 160 len 36864 data: 17 17 17 14 12 11 9 8 7
203402023-09-22T23:33:28.796Z19/30 Write block 555 len 32768 data: 8 9 8 8 9 12 10 10
203412023-09-22T23:33:28.799Z20/30 Read block 501 len 28672
203422023-09-22T23:33:28.801Z21/30 Read block 401 len 12288
203432023-09-22T23:33:28.804Z22/30 Write block 332 len 32768 data: 12 12 16 17 17 13 13 12
203442023-09-22T23:33:28.807Z23/30 Flush
203452023-09-22T23:33:28.810Z24/30 Write block 485 len 32768 data: 17 20 17 21 22 19 20 17
203462023-09-22T23:33:28.813Z25/30 Write block 50 len 40960 data: 8 9 7 5 4 5 5 6 10 12
203472023-09-22T23:33:28.816ZError: Difference in extent metadata found!
203482023-09-22T23:33:28.818Z26/30 Read block 495 len 12288
203492023-09-22T23:33:28.821Z27/30 Write block 356 len 36864 data: 12 11 13 16 17 13 10 8 7
203502023-09-22T23:33:28.824Z28/30 Read block 508 len 24576
203512023-09-22T23:33:28.827Z29/30 Write block 435 len 8192 data: 17 16
203522023-09-22T23:33:28.830Z30/30 Read block 432 len 24576
203532023-09-22T23:33:28.832Z----------------------------------------------------------------
203542023-09-22T23:33:28.835Z Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30
203552023-09-22T23:33:28.838ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
203562023-09-22T23:33:28.841Z 1 Acked 1000 Write 6 Done Done Done false
203572023-09-22T23:33:28.844Z 2 Acked 1001 Write 4 Sent Done Done false
203582023-09-22T23:33:28.847Z 3 Acked 1002 Flush 0 Sent Done Done false
203592023-09-22T23:33:28.850Z 4 Acked 1003 Write 10 Sent Done Done false
203602023-09-22T23:33:28.853Z 5 Acked 1004 Read 6 Sent Done Done false
203612023-09-22T23:33:28.856Z 6 Acked 1005 Write 5 Sent Done Done false
203622023-09-22T23:33:28.859Z 7 Acked 1006 Flush 0 Sent Done Done false
203632023-09-22T23:33:28.862Z 8 Acked 1007 Flush 0 Sent Done Done false
203642023-09-22T23:33:28.865Z 9 Acked 1008 Flush 0 Sent Done Done false
203652023-09-22T23:33:28.868Z 10 Acked 1009 Write 4 Sent Done Done false
203662023-09-22T23:33:28.870Z 11 Acked 1010 Read 1 Sent Done Done false
203672023-09-22T23:33:28.873Z 12 Acked 1011 Read 8 Sent Done Done false
203682023-09-22T23:33:28.876Z 13 Acked 1012 Read 8 Sent Done Done false
203692023-09-22T23:33:28.879Z 14 Acked 1013 Read 2 Sent Done Done false
203702023-09-22T23:33:28.882Z 15 Acked 1014 Read 6 Sent Done Done false
203712023-09-22T23:33:28.885Z 16 Acked 1015 Read 3 Sent Done Done false
203722023-09-22T23:33:28.888Z 17 Acked 1016 Write 2 Sent Done Done false
203732023-09-22T23:33:28.891Z 18 Acked 1017 Write 9 Sent Done Done false
203742023-09-22T23:33:28.894Z 19 Acked 1018 Write 8 Sent Done Done false
203752023-09-22T23:33:28.897Z 20 Acked 1019 Read 7 Sent Done Done false
203762023-09-22T23:33:28.900Z 21 Acked 1020 Read 3 Sent Done Done false
203772023-09-22T23:33:28.903Z 22 Acked 1021 Write 8 Sent Done Done false
203782023-09-22T23:33:28.906Z 23 Acked 1022 Flush 0 Sent Done Done false
203792023-09-22T23:33:28.909Z 24 Acked 1023 Write 8 Sent Done Done false
203802023-09-22T23:33:28.913Z 25 Acked 1024 Write 10 Sent Done Done false
203812023-09-22T23:33:28.916Z 26 Acked 1025 Read 3 Sent Done Done false
203822023-09-22T23:33:28.919Z 27 Acked 1026 Write 9 Sent Done Done false
203832023-09-22T23:33:28.922Z 28 Acked 1027 Read 6 Sent Done Done false
203842023-09-22T23:33:28.925Z 29 Acked 1028 Write 2 Sent Done Done false
203852023-09-22T23:33:28.928Z 30 Acked 1029 Read 6 Sent Done Sent false
203862023-09-22T23:33:28.931Z STATES DS:0 DS:1 DS:2 TOTAL
203872023-09-22T23:33:28.934Z New 0 0 0 0
203882023-09-22T23:33:28.937Z Sent 29 0 1 30
203892023-09-22T23:33:28.939Z Done 1 30 29 60
203902023-09-22T23:33:28.942Z Skipped 0 0 0 0
203912023-09-22T23:33:28.945Z Error 0 0 0 0
203922023-09-22T23:33:28.948ZLast Flush: 0 1022 1022
203932023-09-22T23:33:28.951ZDownstairs last five completed:
203942023-09-22T23:33:28.954ZUpstairs last five completed: 30 29 28 27 26 25
203952023-09-22T23:33:28.957ZWrote out file "/tmp/test_repair_verify.data"
203962023-09-22T23:33:28.960Z
203972023-09-22T23:33:28.963ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
203982023-09-22T23:33:28.987Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.74134696Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1797}
203992023-09-22T23:33:28.990Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.741609912Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1797}
204002023-09-22T23:33:28.993Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.741632885Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1797}
204012023-09-22T23:33:28.996Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.768583979Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1797}
204022023-09-22T23:33:28.999Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.768612605Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1797}
204032023-09-22T23:33:29.002Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.768630725Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1797}
204042023-09-22T23:33:29.005Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.786882776Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1797}
204052023-09-22T23:33:29.008Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.786910252Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1797}
204062023-09-22T23:33:29.011Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:27.786928615Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1797}
204072023-09-22T23:33:29.014ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
204082023-09-22T23:33:29.017Z 0 000-019 153 153 153 242 242 242 F F F
204092023-09-22T23:33:29.021Z 1 020-039 148 148 148 231 231 231 F F F
204102023-09-22T23:33:29.023Z 2 040-059 144 154 154 224 243 243 F T T <---
204112023-09-22T23:33:29.027Z 3 060-079 153 153 153 242 242 242 F F F
204122023-09-22T23:33:29.030Z 4 080-099 153 154 154 242 244 244 F F F <---
204132023-09-22T23:33:29.033Z 5 100-119 152 154 154 240 247 247 F F F <---
204142023-09-22T23:33:29.036Z 6 120-139 151 151 151 236 236 236 F F F
204152023-09-22T23:33:29.039Z 7 140-159 153 153 153 242 242 242 F F F
204162023-09-22T23:33:29.041Z 8 160-179 147 154 154 228 247 247 F F F <---
204172023-09-22T23:33:29.044Z 9 180-199 150 150 150 235 235 235 F F F
204182023-09-22T23:33:29.047Z 10 200-219 153 153 153 242 242 242 F F F
204192023-09-22T23:33:29.050Z 11 220-239 152 152 152 241 241 241 F F F
204202023-09-22T23:33:29.053Z 12 240-259 148 154 154 231 243 243 T F F <---
204212023-09-22T23:33:29.055Z 13 260-279 149 149 149 232 232 232 F F F
204222023-09-22T23:33:29.058Z 14 280-299 148 148 148 229 229 229 F F F
204232023-09-22T23:33:29.061Z 15 300-319 149 149 149 232 232 232 F F F
204242023-09-22T23:33:29.064Z 16 320-339 150 154 154 235 247 247 F F F <---
204252023-09-22T23:33:29.066Z 17 340-359 150 150 150 235 235 235 F T T <---
204262023-09-22T23:33:29.069Z 18 360-379 150 150 150 235 235 235 F T T <---
204272023-09-22T23:33:29.072Z 19 380-399 150 150 150 235 235 235 F F F
204282023-09-22T23:33:29.075Z 20 400-419 142 142 142 219 219 219 F F F
204292023-09-22T23:33:29.077Z 21 420-439 153 153 153 242 242 242 F T T <---
204302023-09-22T23:33:29.080Z 22 440-459 153 154 154 242 244 244 F F F <---
204312023-09-22T23:33:29.083Z 23 460-479 153 153 153 242 242 242 F F F
204322023-09-22T23:33:29.086Z 24 480-499 149 149 149 232 232 232 F T T <---
204332023-09-22T23:33:29.089Z 25 500-519 152 154 154 240 247 247 F F F <---
204342023-09-22T23:33:29.092Z 26 520-539 152 152 152 238 238 238 F F F
204352023-09-22T23:33:29.094Z 27 540-559 151 154 154 236 247 247 F F F <---
204362023-09-22T23:33:29.097Z 28 560-579 143 154 154 222 247 247 F F F <---
204372023-09-22T23:33:29.100Z 29 580-599 151 151 151 236 236 236 F F F
204382023-09-22T23:33:29.104ZMax gen: 154, Max flush: 247
204392023-09-22T23:33:29.107ZA Difference in extent metadata is expected here
204402023-09-22T23:33:29.110ZOn loop 77
204412023-09-22T23:33:29.113Z
204422023-09-22T23:33:29.115ZVerifying data now
204432023-09-22T23:33:34.292ZNo change after verify
204442023-09-22T23:33:34.295ZLoop: 77 Downstairs dump after verify (and repair):
204452023-09-22T23:33:34.310Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.308034702Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1802}
204462023-09-22T23:33:34.313Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.308394323Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1802}
204472023-09-22T23:33:34.316Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.308424432Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1802}
204482023-09-22T23:33:34.328Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.326931891Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1802}
204492023-09-22T23:33:34.331Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.326984252Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1802}
204502023-09-22T23:33:34.334Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.327004782Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1802}
204512023-09-22T23:33:34.339Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.337865585Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1802}
204522023-09-22T23:33:34.342Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.337918048Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1802}
204532023-09-22T23:33:34.345Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.337938684Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1802}
204542023-09-22T23:33:34.350ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
204552023-09-22T23:33:34.353Z 0 000-019 153 153 153 242 242 242 F F F
204562023-09-22T23:33:34.357Z 1 020-039 148 148 148 231 231 231 F F F
204572023-09-22T23:33:34.360Z 2 040-059 155 155 155 248 248 248 F F F
204582023-09-22T23:33:34.363Z 3 060-079 153 153 153 242 242 242 F F F
204592023-09-22T23:33:34.366Z 4 080-099 154 154 154 244 244 244 F F F
204602023-09-22T23:33:34.369Z 5 100-119 154 154 154 247 247 247 F F F
204612023-09-22T23:33:34.371Z 6 120-139 151 151 151 236 236 236 F F F
204622023-09-22T23:33:34.375Z 7 140-159 153 153 153 242 242 242 F F F
204632023-09-22T23:33:34.378Z 8 160-179 154 154 154 247 247 247 F F F
204642023-09-22T23:33:34.380Z 9 180-199 150 150 150 235 235 235 F F F
204652023-09-22T23:33:34.383Z 10 200-219 153 153 153 242 242 242 F F F
204662023-09-22T23:33:34.386Z 11 220-239 152 152 152 241 241 241 F F F
204672023-09-22T23:33:34.389Z 12 240-259 154 154 154 243 243 243 F F F
204682023-09-22T23:33:34.392Z 13 260-279 149 149 149 232 232 232 F F F
204692023-09-22T23:33:34.395Z 14 280-299 148 148 148 229 229 229 F F F
204702023-09-22T23:33:34.398Z 15 300-319 149 149 149 232 232 232 F F F
204712023-09-22T23:33:34.402Z 16 320-339 154 154 154 247 247 247 F F F
204722023-09-22T23:33:34.405Z 17 340-359 155 155 155 248 248 248 F F F
204732023-09-22T23:33:34.407Z 18 360-379 155 155 155 248 248 248 F F F
204742023-09-22T23:33:34.410Z 19 380-399 150 150 150 235 235 235 F F F
204752023-09-22T23:33:34.413Z 20 400-419 142 142 142 219 219 219 F F F
204762023-09-22T23:33:34.416Z 21 420-439 155 155 155 248 248 248 F F F
204772023-09-22T23:33:34.420Z 22 440-459 154 154 154 244 244 244 F F F
204782023-09-22T23:33:34.422Z 23 460-479 153 153 153 242 242 242 F F F
204792023-09-22T23:33:34.425Z 24 480-499 155 155 155 248 248 248 F F F
204802023-09-22T23:33:34.428Z 25 500-519 154 154 154 247 247 247 F F F
204812023-09-22T23:33:34.431Z 26 520-539 152 152 152 238 238 238 F F F
204822023-09-22T23:33:34.434Z 27 540-559 154 154 154 247 247 247 F F F
204832023-09-22T23:33:34.437Z 28 560-579 154 154 154 247 247 247 F F F
204842023-09-22T23:33:34.440Z 29 580-599 151 151 151 236 236 236 F F F
204852023-09-22T23:33:34.443ZMax gen: 155, Max flush: 248
204862023-09-22T23:33:34.446Z
204872023-09-22T23:33:34.449ZBegin loop 78
204882023-09-22T23:33:34.452ZDownstairs to restart: 2
204892023-09-22T23:33:34.455Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.387141668Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
204902023-09-22T23:33:34.457Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.387515522Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
204912023-09-22T23:33:34.461Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.387568423Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
204922023-09-22T23:33:34.464Z{"msg":"Crucible stats registered with UUID: cfcde241-6499-46e3-b376-92e6cdc00757","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.387611967Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
204932023-09-22T23:33:34.467Z{"msg":"Crucible cfcde241-6499-46e3-b376-92e6cdc00757 has session id: 11b55087-7b44-4e93-b58c-5d2bddff6737","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.387658697Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
204942023-09-22T23:33:34.470ZCrucible runtime is spawned
204952023-09-22T23:33:34.473ZThe guest has requested activation with gen:156
204962023-09-22T23:33:34.476Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.387883184Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804,"task":"up_listen"}
204972023-09-22T23:33:34.479Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.387930426Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
204982023-09-22T23:33:34.481Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.387975774Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
204992023-09-22T23:33:34.484Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.388085193Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804,"looper":"0"}
205002023-09-22T23:33:34.487Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.388400205Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804,"looper":"2"}
205012023-09-22T23:33:34.490Z{"msg":"[0] cfcde241-6499-46e3-b376-92e6cdc00757 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.388487155Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804,"looper":"0"}
205022023-09-22T23:33:34.493Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.388564394Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804,"looper":"1"}
205032023-09-22T23:33:34.496Z{"msg":"[1] cfcde241-6499-46e3-b376-92e6cdc00757 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.388787768Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804,"looper":"1"}
205042023-09-22T23:33:34.499Z{"msg":"cfcde241-6499-46e3-b376-92e6cdc00757 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.388952887Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205052023-09-22T23:33:34.502Z{"msg":"Set desired generation to :156","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.389007586Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205062023-09-22T23:33:34.506Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.38903005Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205072023-09-22T23:33:34.508Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.389299933Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205082023-09-22T23:33:34.511Z{"msg":"[0] cfcde241-6499-46e3-b376-92e6cdc00757 (cb5f39b0-084f-4dc5-b779-90ab08ccb723) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.389730583Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205092023-09-22T23:33:34.515Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.389792595Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205102023-09-22T23:33:34.518Z{"msg":"[0] client is_active_req TRUE, promote! session cb5f39b0-084f-4dc5-b779-90ab08ccb723","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.389815877Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205112023-09-22T23:33:34.521Z{"msg":"[1] cfcde241-6499-46e3-b376-92e6cdc00757 (cb5f39b0-084f-4dc5-b779-90ab08ccb723) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.389998544Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205122023-09-22T23:33:34.524Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.39005646Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205132023-09-22T23:33:34.527Z{"msg":"[1] client is_active_req TRUE, promote! session cb5f39b0-084f-4dc5-b779-90ab08ccb723","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.390079242Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205142023-09-22T23:33:34.530Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.390472901Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205152023-09-22T23:33:34.533Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.390541338Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205162023-09-22T23:33:34.535Z{"msg":"cfcde241-6499-46e3-b376-92e6cdc00757 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.390641781Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205172023-09-22T23:33:34.538Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.39078124Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205182023-09-22T23:33:34.541Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.390860324Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205192023-09-22T23:33:34.546Z{"msg":"cfcde241-6499-46e3-b376-92e6cdc00757 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.39090736Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205202023-09-22T23:33:34.550Z{"msg":"[0] cfcde241-6499-46e3-b376-92e6cdc00757 (cb5f39b0-084f-4dc5-b779-90ab08ccb723) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.392873677Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205212023-09-22T23:33:34.552Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.392958216Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205222023-09-22T23:33:34.555Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:34.393037699Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205232023-09-22T23:33:34.558Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.393111013Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205242023-09-22T23:33:34.561Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.393185671Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205252023-09-22T23:33:34.563Z{"msg":"cfcde241-6499-46e3-b376-92e6cdc00757 WaitQuorum WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.393257582Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205262023-09-22T23:33:34.566Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.393331403Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205272023-09-22T23:33:34.569Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.393404968Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205282023-09-22T23:33:34.572Z{"msg":"[1] cfcde241-6499-46e3-b376-92e6cdc00757 (cb5f39b0-084f-4dc5-b779-90ab08ccb723) WaitQuorum WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.39346042Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205292023-09-22T23:33:34.575Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.393502008Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205302023-09-22T23:33:34.577Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:34.39354839Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205312023-09-22T23:33:34.580Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.393592657Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205322023-09-22T23:33:34.583Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.393615037Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205332023-09-22T23:33:34.586Z{"msg":"cfcde241-6499-46e3-b376-92e6cdc00757 WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.393661707Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205342023-09-22T23:33:34.589Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.393683389Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205352023-09-22T23:33:34.592Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:34.393700783Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205362023-09-22T23:33:35.391Z{"msg":"[2] cfcde241-6499-46e3-b376-92e6cdc00757 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.38927379Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804,"looper":"2"}
205372023-09-22T23:33:35.394Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.389374961Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205382023-09-22T23:33:35.397Z{"msg":"[2] cfcde241-6499-46e3-b376-92e6cdc00757 (cb5f39b0-084f-4dc5-b779-90ab08ccb723) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.390456071Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205392023-09-22T23:33:35.400Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.390496223Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205402023-09-22T23:33:35.404Z{"msg":"[2] client is_active_req TRUE, promote! session cb5f39b0-084f-4dc5-b779-90ab08ccb723","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.390555531Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205412023-09-22T23:33:35.406Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.391250339Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205422023-09-22T23:33:35.409Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.391330552Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205432023-09-22T23:33:35.412Z{"msg":"cfcde241-6499-46e3-b376-92e6cdc00757 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.391383632Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205442023-09-22T23:33:35.415Z{"msg":"[2] cfcde241-6499-46e3-b376-92e6cdc00757 (cb5f39b0-084f-4dc5-b779-90ab08ccb723) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.393695703Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205452023-09-22T23:33:35.418Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.393759334Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205462023-09-22T23:33:35.421Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:35.393810461Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205472023-09-22T23:33:35.424Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.393861217Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205482023-09-22T23:33:35.427Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.39392168Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205492023-09-22T23:33:35.430Z{"msg":"cfcde241-6499-46e3-b376-92e6cdc00757 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.39396679Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205502023-09-22T23:33:35.433Z{"msg":"[0]R flush_numbers[0..12]: [242, 231, 248, 242, 244, 247, 236, 242, 247, 235, 242, 241]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.39400882Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205512023-09-22T23:33:35.436Z{"msg":"[0]R generation[0..12]: [153, 148, 155, 153, 154, 154, 151, 153, 154, 150, 153, 152]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.394049271Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205522023-09-22T23:33:35.439Z{"msg":"The guest has finished waiting for activation with:156
205532023-09-22T23:33:35.442ZWait for a query_work_queue command to finish before sending IO
205542023-09-22T23:33:35.445Z[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.394255248Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205552023-09-22T23:33:35.448Z{"msg":"[1]R flush_numbers[0..12]: [242, 231, 248, 242, 244, 247, 236, 242, 247, 235, 242, 241]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.394377904Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205562023-09-22T23:33:35.451Z{"msg":"[1]R generation[0..12]: [153, 148, 155, 153, 154, 154, 151, 153, 154, 150, 153, 152]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.39443944Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205572023-09-22T23:33:35.454Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.394511738Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205582023-09-22T23:33:35.460Z{"msg":"[2]R flush_numbers[0..12]: [242, 231, 248, 242, 244, 247, 236, 242, 247, 235, 242, 241]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.394556787Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205592023-09-22T23:33:35.463Z{"msg":"[2]R generation[0..12]: [153, 148, 155, 153, 154, 154, 151, 153, 154, 150, 153, 152]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.394597965Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205602023-09-22T23:33:35.466Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.394695935Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205612023-09-22T23:33:35.469Z{"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.394741006Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205622023-09-22T23:33:35.472Z{"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.394780898Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205632023-09-22T23:33:35.475Z{"msg":"Next flush: 249","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.394829343Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205642023-09-22T23:33:35.477Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.394876922Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205652023-09-22T23:33:35.480Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.394915415Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205662023-09-22T23:33:35.483Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.394954615Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205672023-09-22T23:33:35.486Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.395000222Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205682023-09-22T23:33:35.489Z{"msg":"cfcde241-6499-46e3-b376-92e6cdc00757 is now active with session: cb5f39b0-084f-4dc5-b779-90ab08ccb723","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.395049017Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205692023-09-22T23:33:35.492Z{"msg":"cfcde241-6499-46e3-b376-92e6cdc00757 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.395096982Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205702023-09-22T23:33:35.494Z{"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
205712023-09-22T23:33:35.497ZNotify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.395144228Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205722023-09-22T23:33:35.500Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.39518309Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205732023-09-22T23:33:35.503Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.395221586Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205742023-09-22T23:33:35.506Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.395259788Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205752023-09-22T23:33:35.510Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.395297881Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205762023-09-22T23:33:35.513Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.395335851Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205772023-09-22T23:33:35.516Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.395374628Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205782023-09-22T23:33:35.519Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.395412652Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205792023-09-22T23:33:35.522Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.395450665Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205802023-09-22T23:33:35.525Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.395488586Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205812023-09-22T23:33:35.528Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.395526608Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1804}
205822023-09-22T23:33:35.531ZLoading write count information from file "/tmp/test_repair_verify.data"
205832023-09-22T23:33:35.534ZRun Repair workload
205842023-09-22T23:33:35.537Z01/30 Write block 460 len 28672 data: 16 15 13 13 12 16 13
205852023-09-22T23:33:35.539Z02/30 Write block 83 len 20480 data: 11 10 8 9 13
205862023-09-22T23:33:35.542Z03/30 Read block 397 len 16384
205872023-09-22T23:33:35.545Z04/30 Read block 205 len 24576
205882023-09-22T23:33:35.548Z05/30 Read block 182 len 24576
205892023-09-22T23:33:35.551Z06/30 Read block 202 len 36864
205902023-09-22T23:33:35.554Z07/30 Read block 358 len 36864
205912023-09-22T23:33:35.559Z08/30 Read block 45 len 28672
205922023-09-22T23:33:35.563Z09/30 Read block 4 len 24576
205932023-09-22T23:33:35.565Z10/30 Flush
205942023-09-22T23:33:35.568Z11/30 Write block 394 len 40960 data: 12 15 15 13 14 12 15 12 14 13
205952023-09-22T23:33:35.571Z12/30 Write block 411 len 12288 data: 12 10 10
205962023-09-22T23:33:35.574Z13/30 Read block 320 len 28672
205972023-09-22T23:33:35.577Z14/30 Read block 44 len 24576
205982023-09-22T23:33:35.580Z15/30 Write block 358 len 4096 data: 14
205992023-09-22T23:33:35.583Z16/30 Write block 68 len 16384 data: 22 20 12 13
206002023-09-22T23:33:35.586Z17/30 Write block 70 len 8192 data: 13 14
206012023-09-22T23:33:35.589Z18/30 Read block 550 len 20480
206022023-09-22T23:33:35.592Z19/30 Read block 430 len 40960
206032023-09-22T23:33:35.595Z20/30 Read block 356 len 4096
206042023-09-22T23:33:35.598Z21/30 Write block 394 len 16384 data: 13 16 16 14
206052023-09-22T23:33:35.601Z22/30 Read block 509 len 16384
206062023-09-22T23:33:35.604Z23/30 Write block 30 len 20480 data: 6 8 6 6 10
206072023-09-22T23:33:35.607Z24/30 Write block 179 len 40960 data: 13 11 12 12 9 11 10 12 12 10
206082023-09-22T23:33:35.610Z25/30 Write block 388 len 12288 data: 12 12 12
206092023-09-22T23:33:35.613Z26/30 Read block 418 len 8192
206102023-09-22T23:33:35.616Z27/30 Read block 343 len 4096
206112023-09-22T23:33:35.619Z28/30 Read block 168 len 32768
206122023-09-22T23:33:35.622Z29/30 Write block 145 len 12288 data: 8 8 8
206132023-09-22T23:33:35.625Z30/30 Read block 493 len 28672
206142023-09-22T23:33:35.628Z----------------------------------------------------------------
206152023-09-22T23:33:35.631Z Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:30
206162023-09-22T23:33:35.634ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
206172023-09-22T23:33:35.637Z 1 Acked 1000 Write 7 Done Done Sent false
206182023-09-22T23:33:35.640Z 2 Acked 1001 Write 5 Done Done Sent false
206192023-09-22T23:33:35.643Z 3 Acked 1002 Read 4 Done Done Sent false
206202023-09-22T23:33:35.646Z 4 Acked 1003 Read 6 Done Done Sent false
206212023-09-22T23:33:35.649Z 5 Acked 1004 Read 6 Done Done Sent false
206222023-09-22T23:33:35.652Z 6 Acked 1005 Read 9 Done Done Sent false
206232023-09-22T23:33:35.654Z 7 Acked 1006 Read 9 Done Done Sent false
206242023-09-22T23:33:35.657Z 8 Acked 1007 Read 7 Done Done Sent false
206252023-09-22T23:33:35.660Z 9 Acked 1008 Read 6 Done Done Sent false
206262023-09-22T23:33:35.663Z 10 Acked 1009 Flush 0 Done Done Sent false
206272023-09-22T23:33:35.665Z 11 Acked 1010 Write 10 Done Done Sent false
206282023-09-22T23:33:35.668Z 12 Acked 1011 Write 3 Done Done Sent false
206292023-09-22T23:33:35.671Z 13 Acked 1012 Read 7 Done Done Sent false
206302023-09-22T23:33:35.674Z 14 Acked 1013 Read 6 Done Done Sent false
206312023-09-22T23:33:35.676Z 15 Acked 1014 Write 1 Done Done Sent false
206322023-09-22T23:33:35.679Z 16 Acked 1015 Write 4 Done Done Sent false
206332023-09-22T23:33:35.682Z 17 Acked 1016 Write 2 Done Done Sent false
206342023-09-22T23:33:35.685Z 18 Acked 1017 Read 5 Done Done Sent false
206352023-09-22T23:33:35.687Z 19 Acked 1018 Read 10 Done Done Sent false
206362023-09-22T23:33:35.690Z 20 Acked 1019 Read 1 Done Done Sent false
206372023-09-22T23:33:35.693Z 21 Acked 1020 Write 4 Done Done Sent false
206382023-09-22T23:33:35.696Z 22 Acked 1021 Read 4 Done Done Sent false
206392023-09-22T23:33:35.699Z 23 Acked 1022 Write 5 Done Done Sent false
206402023-09-22T23:33:35.701Z 24 Acked 1023 Write 10 Done Done Sent false
206412023-09-22T23:33:35.705Z 25 Acked 1024 Write 3 Done Done Sent false
206422023-09-22T23:33:35.708Z 26 Acked 1025 Read 2 Done Done Sent false
206432023-09-22T23:33:35.710Z 27 Acked 1026 Read 1 Done Done Sent false
206442023-09-22T23:33:35.713Z 28 Acked 1027 Read 8 Done Done Sent false
206452023-09-22T23:33:35.717Z 29 Acked 1028 Write 3 Done Done Sent false
206462023-09-22T23:33:35.720Z 30 Acked 1029 Read 7 Done Done Sent false
206472023-09-22T23:33:35.722Z STATES DS:0 DS:1 DS:2 TOTAL
206482023-09-22T23:33:35.726Z New 0 0 0 0
206492023-09-22T23:33:35.729Z Sent 0 0 30 30
206502023-09-22T23:33:35.732Z Done 30 30 0 60
206512023-09-22T23:33:35.735Z Skipped 0 0 0 0
206522023-09-22T23:33:35.738Z Error 0 0 0 0
206532023-09-22T23:33:35.741ZLast Flush: 1009 1009 0
206542023-09-22T23:33:35.743ZDownstairs last five completed:
206552023-09-22T23:33:35.746ZUpstairs last five completed: 30 29 28 27 26 25
206562023-09-22T23:33:35.749ZWrote out file "/tmp/test_repair_verify.data"
206572023-09-22T23:33:35.752Z
206582023-09-22T23:33:35.755ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
206592023-09-22T23:33:35.758Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.522252534Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1805}
206602023-09-22T23:33:35.760Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.52251741Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1805}
206612023-09-22T23:33:35.763Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.522540876Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1805}
206622023-09-22T23:33:35.766Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.553645416Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1805}
206632023-09-22T23:33:35.769Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.55367804Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1805}
206642023-09-22T23:33:35.772Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.553697484Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1805}
206652023-09-22T23:33:35.774Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.574920436Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1805}
206662023-09-22T23:33:35.777Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.574951383Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1805}
206672023-09-22T23:33:35.780ZError: Difference in extent metadata found!
206682023-09-22T23:33:35.784Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:35.57497043Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1805}
206692023-09-22T23:33:35.787ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
206702023-09-22T23:33:35.790Z 0 000-019 153 153 153 242 242 242 F F F
206712023-09-22T23:33:35.793Z 1 020-039 148 148 148 231 231 231 T T F <---
206722023-09-22T23:33:35.796Z 2 040-059 155 155 155 248 248 248 F F F
206732023-09-22T23:33:35.799Z 3 060-079 153 153 153 242 242 242 T T F <---
206742023-09-22T23:33:35.802Z 4 080-099 156 156 154 249 249 244 F F F <---
206752023-09-22T23:33:35.805Z 5 100-119 154 154 154 247 247 247 F F F
206762023-09-22T23:33:35.808Z 6 120-139 151 151 151 236 236 236 F F F
206772023-09-22T23:33:35.811Z 7 140-159 153 153 153 242 242 242 T T F <---
206782023-09-22T23:33:35.814Z 8 160-179 154 154 154 247 247 247 T T F <---
206792023-09-22T23:33:35.817Z 9 180-199 150 150 150 235 235 235 T T F <---
206802023-09-22T23:33:35.820Z 10 200-219 153 153 153 242 242 242 F F F
206812023-09-22T23:33:35.823Z 11 220-239 152 152 152 241 241 241 F F F
206822023-09-22T23:33:35.825Z 12 240-259 154 154 154 243 243 243 F F F
206832023-09-22T23:33:35.828Z 13 260-279 149 149 149 232 232 232 F F F
206842023-09-22T23:33:35.831Z 14 280-299 148 148 148 229 229 229 F F F
206852023-09-22T23:33:35.834Z 15 300-319 149 149 149 232 232 232 F F F
206862023-09-22T23:33:35.837Z 16 320-339 154 154 154 247 247 247 F F F
206872023-09-22T23:33:35.840Z 17 340-359 155 155 155 248 248 248 T T F <---
206882023-09-22T23:33:35.843Z 18 360-379 155 155 155 248 248 248 F F F
206892023-09-22T23:33:35.846Z 19 380-399 150 150 150 235 235 235 T T F <---
206902023-09-22T23:33:35.849Z 20 400-419 142 142 142 219 219 219 T T F <---
206912023-09-22T23:33:35.852Z 21 420-439 155 155 155 248 248 248 F F F
206922023-09-22T23:33:35.855Z 22 440-459 154 154 154 244 244 244 F F F
206932023-09-22T23:33:35.858Z 23 460-479 156 156 153 249 249 242 F F F <---
206942023-09-22T23:33:35.861Z 24 480-499 155 155 155 248 248 248 F F F
206952023-09-22T23:33:35.864Z 25 500-519 154 154 154 247 247 247 F F F
206962023-09-22T23:33:35.867Z 26 520-539 152 152 152 238 238 238 F F F
206972023-09-22T23:33:35.870Z 27 540-559 154 154 154 247 247 247 F F F
206982023-09-22T23:33:35.873Z 28 560-579 154 154 154 247 247 247 F F F
206992023-09-22T23:33:35.876Z 29 580-599 151 151 151 236 236 236 F F F
207002023-09-22T23:33:35.879ZMax gen: 156, Max flush: 249
207012023-09-22T23:33:35.882ZA Difference in extent metadata is expected here
207022023-09-22T23:33:35.885ZOn loop 78
207032023-09-22T23:33:35.888Z
207042023-09-22T23:33:35.891ZVerifying data now
207052023-09-22T23:33:41.760ZNo change after verify
207062023-09-22T23:33:41.763ZLoop: 78 Downstairs dump after verify (and repair):
207072023-09-22T23:33:41.778Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.776608746Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1810}
207082023-09-22T23:33:41.782Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.776924971Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1810}
207092023-09-22T23:33:41.785Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.776949318Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1810}
207102023-09-22T23:33:41.797Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.795665476Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1810}
207112023-09-22T23:33:41.800Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.795718538Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1810}
207122023-09-22T23:33:41.804Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.795738873Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1810}
207132023-09-22T23:33:41.808Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.806548095Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1810}
207142023-09-22T23:33:41.811Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.806599044Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1810}
207152023-09-22T23:33:41.814Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.806619393Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1810}
207162023-09-22T23:33:41.819ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
207172023-09-22T23:33:41.822Z 0 000-019 153 153 153 242 242 242 F F F
207182023-09-22T23:33:41.825Z 1 020-039 157 157 157 250 250 250 F F F
207192023-09-22T23:33:41.828Z 2 040-059 155 155 155 248 248 248 F F F
207202023-09-22T23:33:41.831Z 3 060-079 157 157 157 250 250 250 F F F
207212023-09-22T23:33:41.834Z 4 080-099 156 156 156 249 249 249 F F F
207222023-09-22T23:33:41.837Z 5 100-119 154 154 154 247 247 247 F F F
207232023-09-22T23:33:41.840Z 6 120-139 151 151 151 236 236 236 F F F
207242023-09-22T23:33:41.844Z 7 140-159 157 157 157 250 250 250 F F F
207252023-09-22T23:33:41.847Z 8 160-179 157 157 157 250 250 250 F F F
207262023-09-22T23:33:41.850Z 9 180-199 157 157 157 250 250 250 F F F
207272023-09-22T23:33:41.853Z 10 200-219 153 153 153 242 242 242 F F F
207282023-09-22T23:33:41.856Z 11 220-239 152 152 152 241 241 241 F F F
207292023-09-22T23:33:41.859Z 12 240-259 154 154 154 243 243 243 F F F
207302023-09-22T23:33:41.862Z 13 260-279 149 149 149 232 232 232 F F F
207312023-09-22T23:33:41.865Z 14 280-299 148 148 148 229 229 229 F F F
207322023-09-22T23:33:41.868Z 15 300-319 149 149 149 232 232 232 F F F
207332023-09-22T23:33:41.871Z 16 320-339 154 154 154 247 247 247 F F F
207342023-09-22T23:33:41.874Z 17 340-359 157 157 157 250 250 250 F F F
207352023-09-22T23:33:41.877Z 18 360-379 155 155 155 248 248 248 F F F
207362023-09-22T23:33:41.880Z 19 380-399 157 157 157 250 250 250 F F F
207372023-09-22T23:33:41.883Z 20 400-419 157 157 157 250 250 250 F F F
207382023-09-22T23:33:41.886Z 21 420-439 155 155 155 248 248 248 F F F
207392023-09-22T23:33:41.889Z 22 440-459 154 154 154 244 244 244 F F F
207402023-09-22T23:33:41.891Z 23 460-479 156 156 156 249 249 249 F F F
207412023-09-22T23:33:41.894Z 24 480-499 155 155 155 248 248 248 F F F
207422023-09-22T23:33:41.897Z 25 500-519 154 154 154 247 247 247 F F F
207432023-09-22T23:33:41.900Z 26 520-539 152 152 152 238 238 238 F F F
207442023-09-22T23:33:41.903Z 27 540-559 154 154 154 247 247 247 F F F
207452023-09-22T23:33:41.906Z 28 560-579 154 154 154 247 247 247 F F F
207462023-09-22T23:33:41.909Z 29 580-599 151 151 151 236 236 236 F F F
207472023-09-22T23:33:41.912ZMax gen: 157, Max flush: 250
207482023-09-22T23:33:41.914Z
207492023-09-22T23:33:41.917ZBegin loop 79
207502023-09-22T23:33:41.920ZDownstairs to restart: 2
207512023-09-22T23:33:41.923Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.854729162Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207522023-09-22T23:33:41.926Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.855109685Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207532023-09-22T23:33:41.929Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.855187626Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207542023-09-22T23:33:41.932Z{"msg":"Crucible stats registered with UUID: ff920b20-818a-4ae4-aeed-11846ba8db9b","v":0,"name":"crucible","level":30Crucible runtime is spawned
207552023-09-22T23:33:41.934Z,"time":"2023-09-22T23:33:41.855265504Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207562023-09-22T23:33:41.937Z{"msg":"Crucible ff920b20-818a-4ae4-aeed-11846ba8db9b has session id: 74345099-8274-4956-a547-2f8df3decf1a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.855401936Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207572023-09-22T23:33:41.940Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30The guest has requested activation with gen:158
207582023-09-22T23:33:41.944Z,"time":"2023-09-22T23:33:41.855441381Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812,"task":"up_listen"}
207592023-09-22T23:33:41.947Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.855528706Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207602023-09-22T23:33:41.950Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.855574935Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207612023-09-22T23:33:41.953Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.855614205Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812,"looper":"1"}
207622023-09-22T23:33:41.956Z{"msg":"[1] ff920b20-818a-4ae4-aeed-11846ba8db9b looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.855902818Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812,"looper":"1"}
207632023-09-22T23:33:41.959Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.855957666Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812,"looper":"2"}
207642023-09-22T23:33:41.961Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.856251474Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812,"looper":"0"}
207652023-09-22T23:33:41.965Z{"msg":"[0] ff920b20-818a-4ae4-aeed-11846ba8db9b looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.856503494Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812,"looper":"0"}
207662023-09-22T23:33:41.968Z{"msg":"ff920b20-818a-4ae4-aeed-11846ba8db9b active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.856761919Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207672023-09-22T23:33:41.970Z{"msg":"Set desired generation to :158","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.856820864Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207682023-09-22T23:33:41.973Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.856861771Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207692023-09-22T23:33:41.976Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.857192093Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207702023-09-22T23:33:41.979Z{"msg":"[1] ff920b20-818a-4ae4-aeed-11846ba8db9b (eadff551-663c-4b4e-bfcc-31077404e76d) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.857689438Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207712023-09-22T23:33:41.982Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.857796515Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207722023-09-22T23:33:41.985Z{"msg":"[1] client is_active_req TRUE, promote! session eadff551-663c-4b4e-bfcc-31077404e76d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.85787292Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207732023-09-22T23:33:41.988Z{"msg":"[0] ff920b20-818a-4ae4-aeed-11846ba8db9b (eadff551-663c-4b4e-bfcc-31077404e76d) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.857952827Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207742023-09-22T23:33:41.991Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.858026733Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207752023-09-22T23:33:41.993Z{"msg":"[0] client is_active_req TRUE, promote! session eadff551-663c-4b4e-bfcc-31077404e76d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.858107002Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207762023-09-22T23:33:41.997Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.858337Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207772023-09-22T23:33:42.000Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.858418531Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207782023-09-22T23:33:42.003Z{"msg":"ff920b20-818a-4ae4-aeed-11846ba8db9b WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.858476204Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207792023-09-22T23:33:42.006Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.85853494Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207802023-09-22T23:33:42.009Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.858598947Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207812023-09-22T23:33:42.012Z{"msg":"ff920b20-818a-4ae4-aeed-11846ba8db9b WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.858654917Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207822023-09-22T23:33:42.015Z{"msg":"[0] ff920b20-818a-4ae4-aeed-11846ba8db9b (eadff551-663c-4b4e-bfcc-31077404e76d) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.861004311Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207832023-09-22T23:33:42.018Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.861130385Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207842023-09-22T23:33:42.020Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:41.861219006Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207852023-09-22T23:33:42.024Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.861310683Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207862023-09-22T23:33:42.026Z{"msg":"[1] ff920b20-818a-4ae4-aeed-11846ba8db9b (eadff551-663c-4b4e-bfcc-31077404e76d) WaitQuorum WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.861388986Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207872023-09-22T23:33:42.032Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.861463075Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207882023-09-22T23:33:42.037Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:41.861535403Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207892023-09-22T23:33:42.040Z{"msg":"ff920b20-818a-4ae4-aeed-11846ba8db9b WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.861609128Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207902023-09-22T23:33:42.043Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.86167877Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207912023-09-22T23:33:42.046Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.861747763Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207922023-09-22T23:33:42.049Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.861815207Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207932023-09-22T23:33:42.052Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.86188001Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207942023-09-22T23:33:42.055Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.861948598Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207952023-09-22T23:33:42.057Z{"msg":"ff920b20-818a-4ae4-aeed-11846ba8db9b WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.862012839Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207962023-09-22T23:33:42.060Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.862079782Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207972023-09-22T23:33:42.063Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:41.86211813Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
207982023-09-22T23:33:42.861Z{"msg":"[2] ff920b20-818a-4ae4-aeed-11846ba8db9b looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.858487524Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812,"looper":"2"}
207992023-09-22T23:33:42.864Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.858585821Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208002023-09-22T23:33:42.867Z{"msg":"[2] ff920b20-818a-4ae4-aeed-11846ba8db9b (eadff551-663c-4b4e-bfcc-31077404e76d) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.859670077Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208012023-09-22T23:33:42.871Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.859725029Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208022023-09-22T23:33:42.874Z{"msg":"[2] client is_active_req TRUE, promote! session eadff551-663c-4b4e-bfcc-31077404e76d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.859768101Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208032023-09-22T23:33:42.877Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.860366934Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208042023-09-22T23:33:42.880Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.860423142Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208052023-09-22T23:33:42.883Z{"msg":"ff920b20-818a-4ae4-aeed-11846ba8db9b WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.860471115Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208062023-09-22T23:33:42.886Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.862824215Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208072023-09-22T23:33:42.889Z{"msg":"[2] ff920b20-818a-4ae4-aeed-11846ba8db9b (eadff551-663c-4b4e-bfcc-31077404e76d) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.863004179Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208082023-09-22T23:33:42.892Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.863070931Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208092023-09-22T23:33:42.895Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:42.86312765Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208102023-09-22T23:33:42.898Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.863183385Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208112023-09-22T23:33:42.901Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.863238466Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208122023-09-22T23:33:42.905Z{"msg":"ff920b20-818a-4ae4-aeed-11846ba8db9b WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.863317536Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208132023-09-22T23:33:42.907Z{"msg":"[0]R flush_numbers[0..12]: [242, 250, 248, 250, 249, 247, 236, 250, 250, 250, 242, 241]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:158
208142023-09-22T23:33:42.911ZWait for a query_work_queue command to finish before sending IO
208152023-09-22T23:33:42.914Z,"time":"2023-09-22T23:33:42.863373729Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208162023-09-22T23:33:42.917Z{"msg":"[0]R generation[0..12]: [153, 157, 155, 157, 156, 154, 151, 157, 157, 157, 153, 152]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.86354866Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208172023-09-22T23:33:42.920Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.863602859Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208182023-09-22T23:33:42.923Z{"msg":"[1]R flush_numbers[0..12]: [242, 250, 248, 250, 249, 247, 236, 250, 250, 250, 242, 241]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.863690738Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208192023-09-22T23:33:42.926Z{"msg":"[1]R generation[0..12]: [153, 157, 155, 157, 156, 154, 151, 157, 157, 157, 153, 152]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.863764986Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208202023-09-22T23:33:42.929Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.863812501Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208212023-09-22T23:33:42.932Z{"msg":"[2]R flush_numbers[0..12]: [242, 250, 248, 250, 249, 247, 236, 250, 250, 250, 242, 241]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.863861469Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208222023-09-22T23:33:42.935Z{"msg":"[2]R generation[0..12]: [153, 157, 155, 157, 156, 154, 151, 157, 157, 157, 153, 152]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.863920486Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208232023-09-22T23:33:42.938Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.863976905Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208242023-09-22T23:33:42.941Z{"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.864030453Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208252023-09-22T23:33:42.944Z{"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.864094526Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208262023-09-22T23:33:42.947Z{"msg":"Next flush: 251","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.864149196Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208272023-09-22T23:33:42.951Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.86418338Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208282023-09-22T23:33:42.954Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.864245703Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208292023-09-22T23:33:42.958Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.864332984Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208302023-09-22T23:33:42.961Z{"msg":"Set Downstairs and Upstairs active","vRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
208312023-09-22T23:33:42.964Z":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.864481029Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208322023-09-22T23:33:42.967Z{"msg":"ff920b20-818a-4ae4-aeed-11846ba8db9b is now active with session: eadff551-663c-4b4e-bfcc-31077404e76d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.864535685Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208332023-09-22T23:33:42.970Z{"msg":"ff920b20-818a-4ae4-aeed-11846ba8db9b Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.864577073Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208342023-09-22T23:33:42.973Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.864615893Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208352023-09-22T23:33:42.976Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.864653423Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208362023-09-22T23:33:42.979Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.86469141Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208372023-09-22T23:33:42.982Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.864729421Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208382023-09-22T23:33:42.985Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.864767332Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208392023-09-22T23:33:42.989Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.864805043Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208402023-09-22T23:33:42.992Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.864842045Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208412023-09-22T23:33:42.995Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.864879262Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208422023-09-22T23:33:42.998Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.864916807Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208432023-09-22T23:33:43.002Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.86496455Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208442023-09-22T23:33:43.005Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.865002112Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1812}
208452023-09-22T23:33:43.008ZLoading write count information from file "/tmp/test_repair_verify.data"
208462023-09-22T23:33:43.018ZRun Repair workload
208472023-09-22T23:33:43.021Z01/30 Write block 475 len 16384 data: 8 9 8 10
208482023-09-22T23:33:43.024Z02/30 Flush
208492023-09-22T23:33:43.027Z03/30 Write block 325 len 32768 data: 6 8 8 12 12 9 12 13
208502023-09-22T23:33:43.029Z04/30 Read block 177 len 12288
208512023-09-22T23:33:43.033Z05/30 Write block 57 len 32768 data: 7 11 13 16 16 17 20 19
208522023-09-22T23:33:43.040Z06/30 Read block 317 len 28672
208532023-09-22T23:33:43.043Z07/30 Write block 418 len 12288 data: 14 15 13
208542023-09-22T23:33:43.046Z08/30 Read block 253 len 40960
208552023-09-22T23:33:43.050Z09/30 Write block 400 len 24576 data: 16 13 15 14 13 11
208562023-09-22T23:33:43.053Z10/30 Write block 404 len 16384 data: 14 12 10 10
208572023-09-22T23:33:43.055Z11/30 Read block 150 len 16384
208582023-09-22T23:33:43.058Z12/30 Flush
208592023-09-22T23:33:43.061Z13/30 Write block 241 len 4096 data: 17
208602023-09-22T23:33:43.064Z14/30 Write block 442 len 8192 data: 16 19
208612023-09-22T23:33:43.067Z15/30 Flush
208622023-09-22T23:33:43.070Z16/30 Write block 466 len 36864 data: 14 13 14 14 11 9 10 9 9
208632023-09-22T23:33:43.073Z17/30 Read block 213 len 8192
208642023-09-22T23:33:43.076Z18/30 Flush
208652023-09-22T23:33:43.079Z19/30 Write block 273 len 8192 data: 17 16
208662023-09-22T23:33:43.082Z20/30 Write block 57 len 4096 data: 8
208672023-09-22T23:33:43.086Z21/30 Read block 396 len 16384
208682023-09-22T23:33:43.090Z22/30 Read block 152 len 4096
208692023-09-22T23:33:43.093Z23/30 Read block 258 len 20480
208702023-09-22T23:33:43.096Z24/30 Read block 442 len 4096
208712023-09-22T23:33:43.099Z25/30 Write block 42 len 32768 data: 11 10 11 12 11 12 10 10
208722023-09-22T23:33:43.102Z26/30 Read block 26 len 8192
208732023-09-22T23:33:43.105Z27/30 Read block 32 len 20480
208742023-09-22T23:33:43.108Z28/30 Read block 351 len 36864
208752023-09-22T23:33:43.111Z29/30 Read block 365 len 36864
208762023-09-22T23:33:43.114Z30/30 Read block 10 len 32768
208772023-09-22T23:33:43.117Z----------------------------------------------------------------
208782023-09-22T23:33:43.120Z Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30
208792023-09-22T23:33:43.123ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
208802023-09-22T23:33:43.126Z 1 Acked 1000 Write 4 Done Done Sent false
208812023-09-22T23:33:43.129Z 2 Acked 1001 Flush 0 Done Done Sent false
208822023-09-22T23:33:43.132Z 3 Acked 1002 Write 8 Done Done Sent false
208832023-09-22T23:33:43.135Z 4 Acked 1003 Read 3 Done Done Sent false
208842023-09-22T23:33:43.138Z 5 Acked 1004 Write 8 Done Done Sent false
208852023-09-22T23:33:43.141Z 6 Acked 1005 Read 7 Done Done Sent false
208862023-09-22T23:33:43.144Z 7 Acked 1006 Write 3 Done Done Sent false
208872023-09-22T23:33:43.147Z 8 Acked 1007 Read 10 Done Done Sent false
208882023-09-22T23:33:43.150Z 9 Acked 1008 Write 6 Done Done Sent false
208892023-09-22T23:33:43.153Z 10 Acked 1009 Write 4 Done Done Sent false
208902023-09-22T23:33:43.156Z 11 Acked 1010 Read 4 Done Done Sent false
208912023-09-22T23:33:43.159Z 12 Acked 1011 Flush 0 Done Done Sent false
208922023-09-22T23:33:43.161Z 13 Acked 1012 Write 1 Done Done Sent false
208932023-09-22T23:33:43.165Z 14 Acked 1013 Write 2 Done Done Sent false
208942023-09-22T23:33:43.168Z 15 Acked 1014 Flush 0 Done Done Sent false
208952023-09-22T23:33:43.171Z 16 Acked 1015 Write 9 Done Done Sent false
208962023-09-22T23:33:43.174Z 17 Acked 1016 Read 2 Done Done Sent false
208972023-09-22T23:33:43.177Z 18 Acked 1017 Flush 0 Done Done Sent false
208982023-09-22T23:33:43.180Z 19 Acked 1018 Write 2 Done Done Sent false
208992023-09-22T23:33:43.183Z 20 Acked 1019 Write 1 Done Done Sent false
209002023-09-22T23:33:43.186Z 21 Acked 1020 Read 4 Done Done Sent false
209012023-09-22T23:33:43.189Z 22 Acked 1021 Read 1 Done Done Sent false
209022023-09-22T23:33:43.193Z 23 Acked 1022 Read 5 Done Done Sent false
209032023-09-22T23:33:43.196Z 24 Acked 1023 Read 1 Done Done Sent false
209042023-09-22T23:33:43.199Z 25 Acked 1024 Write 8 Done Done Sent false
209052023-09-22T23:33:43.202Z 26 Acked 1025 Read 2 Done Done Sent false
209062023-09-22T23:33:43.206Z 27 Acked 1026 Read 5 Done Done Sent false
209072023-09-22T23:33:43.211Z 28 Acked 1027 Read 9 Done Done Sent false
209082023-09-22T23:33:43.215Z 29 Acked 1028 Read 9 Done Done Sent false
209092023-09-22T23:33:43.218Z 30 Acked 1029 Read 8 Done Done Sent false
209102023-09-22T23:33:43.221Z STATES DS:0 DS:1 DS:2 TOTAL
209112023-09-22T23:33:43.224Z New 0 0 0 0
209122023-09-22T23:33:43.226Z Sent 0 0 30 30
209132023-09-22T23:33:43.229Z Done 30 30 0 60
209142023-09-22T23:33:43.232Z Skipped 0 0 0 0
209152023-09-22T23:33:43.235Z Error 0 0 0 0
209162023-09-22T23:33:43.238ZLast Flush: 1017 1017 0
209172023-09-22T23:33:43.241ZDownstairs last five completed:
209182023-09-22T23:33:43.244ZUpstairs last five completed: 30 29 28 27 26 25
209192023-09-22T23:33:43.247ZWrote out file "/tmp/test_repair_verify.data"
209202023-09-22T23:33:43.251ZError: Difference in extent metadata found!
209212023-09-22T23:33:43.253Z
209222023-09-22T23:33:43.257ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
209232023-09-22T23:33:43.259Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.993684684Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1813}
209242023-09-22T23:33:43.263Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.994050884Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1813}
209252023-09-22T23:33:43.266Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:42.994074604Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1813}
209262023-09-22T23:33:43.289Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:43.017063193Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1813}
209272023-09-22T23:33:43.293Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:43.017110533Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1813}
209282023-09-22T23:33:43.296Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:43.017132962Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1813}
209292023-09-22T23:33:43.299Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:43.030665298Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1813}
209302023-09-22T23:33:43.302Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:43.030695968Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1813}
209312023-09-22T23:33:43.305Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:43.030714885Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1813}
209322023-09-22T23:33:43.308ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
209332023-09-22T23:33:43.311Z 0 000-019 153 153 153 242 242 242 F F F
209342023-09-22T23:33:43.314Z 1 020-039 157 157 157 250 250 250 F F F
209352023-09-22T23:33:43.317Z 2 040-059 158 158 155 252 252 248 T T F <---
209362023-09-22T23:33:43.320Z 3 060-079 158 158 157 252 252 250 F F F <---
209372023-09-22T23:33:43.323Z 4 080-099 156 156 156 249 249 249 F F F
209382023-09-22T23:33:43.327Z 5 100-119 154 154 154 247 247 247 F F F
209392023-09-22T23:33:43.330Z 6 120-139 151 151 151 236 236 236 F F F
209402023-09-22T23:33:43.333Z 7 140-159 157 157 157 250 250 250 F F F
209412023-09-22T23:33:43.336Z 8 160-179 157 157 157 250 250 250 F F F
209422023-09-22T23:33:43.339Z 9 180-199 157 157 157 250 250 250 F F F
209432023-09-22T23:33:43.342Z 10 200-219 153 153 153 242 242 242 F F F
209442023-09-22T23:33:43.345Z 11 220-239 152 152 152 241 241 241 F F F
209452023-09-22T23:33:43.348Z 12 240-259 158 158 154 253 253 243 F F F <---
209462023-09-22T23:33:43.350Z 13 260-279 149 149 149 232 232 232 T T F <---
209472023-09-22T23:33:43.354Z 14 280-299 148 148 148 229 229 229 F F F
209482023-09-22T23:33:43.356Z 15 300-319 149 149 149 232 232 232 F F F
209492023-09-22T23:33:43.359Z 16 320-339 158 158 154 252 252 247 F F F <---
209502023-09-22T23:33:43.362Z 17 340-359 157 157 157 250 250 250 F F F
209512023-09-22T23:33:43.365Z 18 360-379 155 155 155 248 248 248 F F F
209522023-09-22T23:33:43.368Z 19 380-399 157 157 157 250 250 250 F F F
209532023-09-22T23:33:43.371Z 20 400-419 158 158 157 252 252 250 F F F <---
209542023-09-22T23:33:43.374Z 21 420-439 158 158 155 252 252 248 F F F <---
209552023-09-22T23:33:43.377Z 22 440-459 158 158 154 253 253 244 F F F <---
209562023-09-22T23:33:43.380Z 23 460-479 158 158 156 254 254 249 F F F <---
209572023-09-22T23:33:43.383Z 24 480-499 155 155 155 248 248 248 F F F
209582023-09-22T23:33:43.386Z 25 500-519 154 154 154 247 247 247 F F F
209592023-09-22T23:33:43.389Z 26 520-539 152 152 152 238 238 238 F F F
209602023-09-22T23:33:43.392Z 27 540-559 154 154 154 247 247 247 F F F
209612023-09-22T23:33:43.395Z 28 560-579 154 154 154 247 247 247 F F F
209622023-09-22T23:33:43.398Z 29 580-599 151 151 151 236 236 236 F F F
209632023-09-22T23:33:43.401ZMax gen: 158, Max flush: 254
209642023-09-22T23:33:43.404ZA Difference in extent metadata is expected here
209652023-09-22T23:33:43.407ZOn loop 79
209662023-09-22T23:33:43.410Z
209672023-09-22T23:33:43.413ZVerifying data now
209682023-09-22T23:33:49.109ZNo change after verify
209692023-09-22T23:33:49.112ZLoop: 79 Downstairs dump after verify (and repair):
209702023-09-22T23:33:49.128Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.125850697Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1818}
209712023-09-22T23:33:49.132Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.126193192Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1818}
209722023-09-22T23:33:49.135Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.126217817Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1818}
209732023-09-22T23:33:49.147Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.145188806Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1818}
209742023-09-22T23:33:49.149Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.145246625Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1818}
209752023-09-22T23:33:49.152Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.145266868Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1818}
209762023-09-22T23:33:49.157Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.156249955Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1818}
209772023-09-22T23:33:49.160Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.156301409Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1818}
209782023-09-22T23:33:49.163Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.156321772Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1818}
209792023-09-22T23:33:49.169ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
209802023-09-22T23:33:49.172Z 0 000-019 153 153 153 242 242 242 F F F
209812023-09-22T23:33:49.174Z 1 020-039 157 157 157 250 250 250 F F F
209822023-09-22T23:33:49.177Z 2 040-059 159 159 159 255 255 255 F F F
209832023-09-22T23:33:49.180Z 3 060-079 158 158 158 252 252 252 F F F
209842023-09-22T23:33:49.182Z 4 080-099 156 156 156 249 249 249 F F F
209852023-09-22T23:33:49.185Z 5 100-119 154 154 154 247 247 247 F F F
209862023-09-22T23:33:49.188Z 6 120-139 151 151 151 236 236 236 F F F
209872023-09-22T23:33:49.191Z 7 140-159 157 157 157 250 250 250 F F F
209882023-09-22T23:33:49.193Z 8 160-179 157 157 157 250 250 250 F F F
209892023-09-22T23:33:49.196Z 9 180-199 157 157 157 250 250 250 F F F
209902023-09-22T23:33:49.199Z 10 200-219 153 153 153 242 242 242 F F F
209912023-09-22T23:33:49.202Z 11 220-239 152 152 152 241 241 241 F F F
209922023-09-22T23:33:49.205Z 12 240-259 158 158 158 253 253 253 F F F
209932023-09-22T23:33:49.208Z 13 260-279 159 159 159 255 255 255 F F F
209942023-09-22T23:33:49.211Z 14 280-299 148 148 148 229 229 229 F F F
209952023-09-22T23:33:49.214Z 15 300-319 149 149 149 232 232 232 F F F
209962023-09-22T23:33:49.217Z 16 320-339 158 158 158 252 252 252 F F F
209972023-09-22T23:33:49.220Z 17 340-359 157 157 157 250 250 250 F F F
209982023-09-22T23:33:49.222Z 18 360-379 155 155 155 248 248 248 F F F
209992023-09-22T23:33:49.227Z 19 380-399 157 157 157 250 250 250 F F F
210002023-09-22T23:33:49.230Z 20 400-419 158 158 158 252 252 252 F F F
210012023-09-22T23:33:49.232Z 21 420-439 158 158 158 252 252 252 F F F
210022023-09-22T23:33:49.235Z 22 440-459 158 158 158 253 253 253 F F F
210032023-09-22T23:33:49.238Z 23 460-479 158 158 158 254 254 254 F F F
210042023-09-22T23:33:49.241Z 24 480-499 155 155 155 248 248 248 F F F
210052023-09-22T23:33:49.243Z 25 500-519 154 154 154 247 247 247 F F F
210062023-09-22T23:33:49.246Z 26 520-539 152 152 152 238 238 238 F F F
210072023-09-22T23:33:49.249Z 27 540-559 154 154 154 247 247 247 F F F
210082023-09-22T23:33:49.252Z 28 560-579 154 154 154 247 247 247 F F F
210092023-09-22T23:33:49.254Z 29 580-599 151 151 151 236 236 236 F F F
210102023-09-22T23:33:49.257ZMax gen: 159, Max flush: 255
210112023-09-22T23:33:49.260Z
210122023-09-22T23:33:49.263ZBegin loop 80
210132023-09-22T23:33:49.266ZDownstairs to restart: 2
210142023-09-22T23:33:49.270Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.205441918Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210152023-09-22T23:33:49.273Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.205811873Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210162023-09-22T23:33:49.276Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.205880234Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210172023-09-22T23:33:49.279Z{"msg":"Crucible stats registered with UUID: 16ecec21-418a-4dd8-aa10-86621c0fc1a5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.20593178Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210182023-09-22T23:33:49.282Z{"msg":"Crucible 16ecec21-418a-4dd8-aa10-86621c0fc1a5 has session id: 6b58f32b-1a91-4a33-bb2d-6c26c1f21a88","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.20600218Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210192023-09-22T23:33:49.285ZCrucible runtime is spawned
210202023-09-22T23:33:49.288Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.20616516Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820,"task":"up_listen"}
210212023-09-22T23:33:49.290Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.20627702Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210222023-09-22T23:33:49.293ZThe guest has requested activation with gen:160
210232023-09-22T23:33:49.296Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.206338482Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210242023-09-22T23:33:49.299Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.206394707Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820,"looper":"0"}
210252023-09-22T23:33:49.302Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.206805858Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820,"looper":"2"}
210262023-09-22T23:33:49.305Z{"msg":"[0] 16ecec21-418a-4dd8-aa10-86621c0fc1a5 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.206881038Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820,"looper":"0"}
210272023-09-22T23:33:49.308Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.206978971Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820,"looper":"1"}
210282023-09-22T23:33:49.311Z{"msg":"[1] 16ecec21-418a-4dd8-aa10-86621c0fc1a5 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.207231886Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820,"looper":"1"}
210292023-09-22T23:33:49.314Z{"msg":"16ecec21-418a-4dd8-aa10-86621c0fc1a5 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.207531395Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210302023-09-22T23:33:49.316Z{"msg":"Set desired generation to :160","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.207587786Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210312023-09-22T23:33:49.319Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.207639484Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210322023-09-22T23:33:49.322Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.208012585Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210332023-09-22T23:33:49.325Z{"msg":"[0] 16ecec21-418a-4dd8-aa10-86621c0fc1a5 (65342f94-dda6-440f-855b-6a75c2668e25) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.208420446Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210342023-09-22T23:33:49.329Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.208496735Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210352023-09-22T23:33:49.332Z{"msg":"[0] client is_active_req TRUE, promote! session 65342f94-dda6-440f-855b-6a75c2668e25","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.208520449Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210362023-09-22T23:33:49.335Z{"msg":"[1] 16ecec21-418a-4dd8-aa10-86621c0fc1a5 (65342f94-dda6-440f-855b-6a75c2668e25) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.208732964Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210372023-09-22T23:33:49.338Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.20878759Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210382023-09-22T23:33:49.342Z{"msg":"[1] client is_active_req TRUE, promote! session 65342f94-dda6-440f-855b-6a75c2668e25","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.20881058Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210392023-09-22T23:33:49.344Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.209269442Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210402023-09-22T23:33:49.347Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.2093279Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210412023-09-22T23:33:49.350Z{"msg":"16ecec21-418a-4dd8-aa10-86621c0fc1a5 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.209546278Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210422023-09-22T23:33:49.353Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.209756242Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210432023-09-22T23:33:49.356Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.209810464Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210442023-09-22T23:33:49.359Z{"msg":"16ecec21-418a-4dd8-aa10-86621c0fc1a5 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.209899328Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210452023-09-22T23:33:49.361Z{"msg":"[0] 16ecec21-418a-4dd8-aa10-86621c0fc1a5 (65342f94-dda6-440f-855b-6a75c2668e25) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.211685653Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210462023-09-22T23:33:49.364Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.211743825Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210472023-09-22T23:33:49.367Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:49.211802059Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210482023-09-22T23:33:49.370Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.211859824Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210492023-09-22T23:33:49.373Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.211915554Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210502023-09-22T23:33:49.376Z{"msg":"16ecec21-418a-4dd8-aa10-86621c0fc1a5 WaitQuorum WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.211970652Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210512023-09-22T23:33:49.379Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.212024425Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210522023-09-22T23:33:49.382Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.212077579Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210532023-09-22T23:33:49.385Z{"msg":"[1] 16ecec21-418a-4dd8-aa10-86621c0fc1a5 (65342f94-dda6-440f-855b-6a75c2668e25) WaitQuorum WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.212134253Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210542023-09-22T23:33:49.388Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.212186497Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210552023-09-22T23:33:49.391Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:49.212250625Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210562023-09-22T23:33:49.394Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.21229226Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210572023-09-22T23:33:49.397Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.21236478Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210582023-09-22T23:33:49.400Z{"msg":"16ecec21-418a-4dd8-aa10-86621c0fc1a5 WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.212409188Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210592023-09-22T23:33:49.403Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.212431466Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210602023-09-22T23:33:49.426Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:49.212467732Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210612023-09-22T23:33:50.210Z{"msg":"[2] 16ecec21-418a-4dd8-aa10-86621c0fc1a5 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.208115934Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820,"looper":"2"}
210622023-09-22T23:33:50.212Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.208200831Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210632023-09-22T23:33:50.216Z{"msg":"[2] 16ecec21-418a-4dd8-aa10-86621c0fc1a5 (65342f94-dda6-440f-855b-6a75c2668e25) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.209394378Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210642023-09-22T23:33:50.219Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.209451102Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210652023-09-22T23:33:50.221Z{"msg":"[2] client is_active_req TRUE, promote! session 65342f94-dda6-440f-855b-6a75c2668e25","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.209495269Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210662023-09-22T23:33:50.224Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.210066479Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210672023-09-22T23:33:50.227Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.210124498Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210682023-09-22T23:33:50.230Z{"msg":"16ecec21-418a-4dd8-aa10-86621c0fc1a5 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.2101681Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210692023-09-22T23:33:50.233Z{"msg":"[2] 16ecec21-418a-4dd8-aa10-86621c0fc1a5 (65342f94-dda6-440f-855b-6a75c2668e25) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.212463987Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210702023-09-22T23:33:50.236Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.212521865Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210712023-09-22T23:33:50.239Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:50.212583173Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210722023-09-22T23:33:50.242Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.212640973Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210732023-09-22T23:33:50.245Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.212684483Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210742023-09-22T23:33:50.249Z{"msg":"16ecec21-418a-4dd8-aa10-86621c0fc1a5 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.212724902Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210752023-09-22T23:33:50.252Z{"msg":"[0]R flush_numbers[0..12]: [242, 250, 255, 252, 249, 247, 236, 250, 250, 250, 242, 241]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.212767059Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210762023-09-22T23:33:50.255Z{"msg":"[0]R generation[0..12]: [153, 157, 159, 158, 156, 154, 151, 157, 157, 157, 153, 152]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.212807494Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210772023-09-22T23:33:50.258Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:160
210782023-09-22T23:33:50.260Z,"time":"2023-09-22T23:33:50.212932856Z","hostname":"Wait for a query_work_queue command to finish before sending IO
210792023-09-22T23:33:50.263Zip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210802023-09-22T23:33:50.266Z{"msg":"[1]R flush_numbers[0..12]: [242, 250, 255, 252, 249, 247, 236, 250, 250, 250, 242, 241]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.212965025Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210812023-09-22T23:33:50.269Z{"msg":"[1]R generation[0..12]: [153, 157, 159, 158, 156, 154, 151, 157, 157, 157, 153, 152]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.213020025Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210822023-09-22T23:33:50.272Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.213042161Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210832023-09-22T23:33:50.275Z{"msg":"[2]R flush_numbers[0..12]: [242, 250, 255, 252, 249, 247, 236, 250, 250, 250, 242, 241]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.213060888Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210842023-09-22T23:33:50.277Z{"msg":"[2]R generation[0..12]: [153, 157, 159, 158, 156, 154, 151, 157, 157, 157, 153, 152]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.213113423Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210852023-09-22T23:33:50.281Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.213173468Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210862023-09-22T23:33:50.283Z{"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.213223246Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210872023-09-22T23:33:50.286Z{"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.213330251Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210882023-09-22T23:33:50.289Z{"msg":"Next flush: 256","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.213383779Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210892023-09-22T23:33:50.292Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.213424607Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210902023-09-22T23:33:50.295Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.213463342Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210912023-09-22T23:33:50.298Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.213510102Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210922023-09-22T23:33:50.301Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.213555884Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210932023-09-22T23:33:50.305Z{"msg":"16ecec21-418a-4dd8-aa10-86621c0fc1a5 is now active with session: 65342f94-dda6-440f-855b-6a75c2668e25","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.21359636Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210942023-09-22T23:33:50.307Z{"msg":"16ecec21-418a-4dd8-aa10-86621c0fc1a5 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.213636293Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210952023-09-22T23:33:50.310Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.21367919Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210962023-09-22T23:33:50.313Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.213720203Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210972023-09-22T23:33:50.315Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.213758551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210982023-09-22T23:33:50.319Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.213801111Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
210992023-09-22T23:33:50.322Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.213843219Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
211002023-09-22T23:33:50.324Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.21388132Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
211012023-09-22T23:33:50.327Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.213919504Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
211022023-09-22T23:33:50.330Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
211032023-09-22T23:33:50.333Z,"time":"2023-09-22T23:33:50.213961439Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
211042023-09-22T23:33:50.336Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.214066089Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
211052023-09-22T23:33:50.339Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.214110766Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
211062023-09-22T23:33:50.342Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.214149417Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1820}
211072023-09-22T23:33:50.345ZLoading write count information from file "/tmp/test_repair_verify.data"
211082023-09-22T23:33:50.348ZRun Repair workload
211092023-09-22T23:33:50.351Z01/30 Write block 470 len 36864 data: 12 10 11 10 10 9 10 9 11
211102023-09-22T23:33:50.354Z02/30 Write block 385 len 32768 data: 8 7 12 13 13 13 10 11
211112023-09-22T23:33:50.357Z03/30 Read block 224 len 28672
211122023-09-22T23:33:50.359Z04/30 Write block 285 len 12288 data: 12 12 12
211132023-09-22T23:33:50.362Z05/30 Read block 226 len 16384
211142023-09-22T23:33:50.365Z06/30 Read block 249 len 32768
211152023-09-22T23:33:50.368Z07/30 Flush
211162023-09-22T23:33:50.371Z08/30 Write block 279 len 16384 data: 20 20 18 18
211172023-09-22T23:33:50.374Z09/30 Read block 208 len 12288
211182023-09-22T23:33:50.377Z10/30 Write block 586 len 8192 data: 8 9
211192023-09-22T23:33:50.379Z11/30 Write block 410 len 12288 data: 9 13 11
211202023-09-22T23:33:50.382Z12/30 Read block 154 len 28672
211212023-09-22T23:33:50.385Z13/30 Write block 61 len 32768 data: 17 18 21 20 22 22 21 23
211222023-09-22T23:33:50.388Z14/30 Read block 249 len 4096
211232023-09-22T23:33:50.391Z15/30 Read block 61 len 28672
211242023-09-22T23:33:50.394Z16/30 Write block 573 len 20480 data: 5 5 8 9 11
211252023-09-22T23:33:50.397Z17/30 Write block 538 len 24576 data: 16 16 11 8 9 10
211262023-09-22T23:33:50.400Z18/30 Read block 342 len 24576
211272023-09-22T23:33:50.402Z19/30 Flush
211282023-09-22T23:33:50.406Z20/30 Write block 49 len 12288 data: 11 9 10
211292023-09-22T23:33:50.408Z21/30 Write block 233 len 12288 data: 9 11 12
211302023-09-22T23:33:50.411Z22/30 Read block 186 len 36864
211312023-09-22T23:33:50.414Z23/30 Read block 93 len 28672
211322023-09-22T23:33:50.417Z24/30 Write block 491 len 32768 data: 21 18 17 15 17 17 17 11
211332023-09-22T23:33:50.420Z25/30 Read block 532 len 24576
211342023-09-22T23:33:50.423Z26/30 Read block 361 len 4096
211352023-09-22T23:33:50.446Z27/30 Read block 377 len 12288
211362023-09-22T23:33:50.450Z28/30 Write block 467 len 8192 data: 14 15
211372023-09-22T23:33:50.453Z29/30 Write block 47 len 28672 data: 13 11 12 10 11 8 6
211382023-09-22T23:33:50.459Z30/30 Read block 454 len 4096
211392023-09-22T23:33:50.462Z----------------------------------------------------------------
211402023-09-22T23:33:50.465Z Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30
211412023-09-22T23:33:50.467ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
211422023-09-22T23:33:50.470Z 1 Acked 1000 Write 9 Done Done Done false
211432023-09-22T23:33:50.473Z 2 Acked 1001 Write 8 Done Done Done false
211442023-09-22T23:33:50.476Z 3 Acked 1002 Read 7 Done Done Done false
211452023-09-22T23:33:50.479Z 4 Acked 1003 Write 3 Done Done Sent false
211462023-09-22T23:33:50.481Z 5 Acked 1004 Read 4 Done Done Sent false
211472023-09-22T23:33:50.484Z 6 Acked 1005 Read 8 Done Done Sent false
211482023-09-22T23:33:50.489Z 7 Acked 1006 Flush 0 Done Done Sent false
211492023-09-22T23:33:50.492Z 8 Acked 1007 Write 4 Done Done Sent false
211502023-09-22T23:33:50.495Z 9 Acked 1008 Read 3 Done Done Sent false
211512023-09-22T23:33:50.498Z 10 Acked 1009 Write 2 Done Done Sent false
211522023-09-22T23:33:50.501Z 11 Acked 1010 Write 3 Done Done Sent false
211532023-09-22T23:33:50.504Z 12 Acked 1011 Read 7 Done Done Sent false
211542023-09-22T23:33:50.507Z 13 Acked 1012 Write 8 Done Done Sent false
211552023-09-22T23:33:50.510Z 14 Acked 1013 Read 1 Done Done Sent false
211562023-09-22T23:33:50.512Z 15 Acked 1014 Read 7 Done Done Sent false
211572023-09-22T23:33:50.515Z 16 Acked 1015 Write 5 Done Done Sent false
211582023-09-22T23:33:50.518Z 17 Acked 1016 Write 6 Done Done Sent false
211592023-09-22T23:33:50.521Z 18 Acked 1017 Read 6 Done Done Sent false
211602023-09-22T23:33:50.525Z 19 Acked 1018 Flush 0 Done Done Sent false
211612023-09-22T23:33:50.528Z 20 Acked 1019 Write 3 Done Done Sent false
211622023-09-22T23:33:50.530Z 21 Acked 1020 Write 3 Done Done Sent false
211632023-09-22T23:33:50.533Z 22 Acked 1021 Read 9 Done Done Sent false
211642023-09-22T23:33:50.536Z 23 Acked 1022 Read 7 Done Done Sent false
211652023-09-22T23:33:50.539Z 24 Acked 1023 Write 8 Done Done Sent false
211662023-09-22T23:33:50.541Z 25 Acked 1024 Read 6 Done Done Sent false
211672023-09-22T23:33:50.544Z 26 Acked 1025 Read 1 Done Done Sent false
211682023-09-22T23:33:50.547Z 27 Acked 1026 Read 3 Done Done Sent false
211692023-09-22T23:33:50.550Z 28 Acked 1027 Write 2 Done Done Sent false
211702023-09-22T23:33:50.552Z 29 Acked 1028 Write 7 Done Done Sent false
211712023-09-22T23:33:50.555Z 30 Acked 1029 Read 1 Done Sent Sent false
211722023-09-22T23:33:50.558Z STATES DS:0 DS:1 DS:2 TOTAL
211732023-09-22T23:33:50.561Z New 0 0 0 0
211742023-09-22T23:33:50.563Z Sent 0 1 27 28
211752023-09-22T23:33:50.566Z Done 30 29 3 62
211762023-09-22T23:33:50.569Z Skipped 0 0 0 0
211772023-09-22T23:33:50.572Z Error 0 0 0 0
211782023-09-22T23:33:50.574ZLast Flush: 1018 1018 0
211792023-09-22T23:33:50.577ZDownstairs last five completed:
211802023-09-22T23:33:50.580ZUpstairs last five completed: 30 29 28 27 26 25
211812023-09-22T23:33:50.582ZWrote out file "/tmp/test_repair_verify.data"
211822023-09-22T23:33:50.585Z
211832023-09-22T23:33:50.588ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
211842023-09-22T23:33:50.591Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.349310268Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1821}
211852023-09-22T23:33:50.594Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.349633981Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1821}
211862023-09-22T23:33:50.597Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.349657428Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1821}
211872023-09-22T23:33:50.600Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.375288169Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1821}
211882023-09-22T23:33:50.603Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.375321988Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1821}
211892023-09-22T23:33:50.605Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.375341029Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1821}
211902023-09-22T23:33:50.608Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.391577162Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1821}
211912023-09-22T23:33:50.611Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.391610322Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1821}
211922023-09-22T23:33:50.614Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:50.391629095Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1821}
211932023-09-22T23:33:50.617ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
211942023-09-22T23:33:50.619Z 0 000-019 153 153 153 242 242 242 F F F
211952023-09-22T23:33:50.622Z 1 020-039 157 157 157 250 250 250 F F F
211962023-09-22T23:33:50.625Z 2 040-059 159 159 159 255 255 255 T T F <---
211972023-09-22T23:33:50.628ZError: Difference in extent metadata found!
211982023-09-22T23:33:50.631Z 3 060-079 160 160 158 257 257 252 F F F <---
211992023-09-22T23:33:50.634Z 4 080-099 156 156 156 249 249 249 F F F
212002023-09-22T23:33:50.637Z 5 100-119 154 154 154 247 247 247 F F F
212012023-09-22T23:33:50.640Z 6 120-139 151 151 151 236 236 236 F F F
212022023-09-22T23:33:50.643Z 7 140-159 157 157 157 250 250 250 F F F
212032023-09-22T23:33:50.645Z 8 160-179 157 157 157 250 250 250 F F F
212042023-09-22T23:33:50.648Z 9 180-199 157 157 157 250 250 250 F F F
212052023-09-22T23:33:50.651Z 10 200-219 153 153 153 242 242 242 F F F
212062023-09-22T23:33:50.654Z 11 220-239 152 152 152 241 241 241 T T F <---
212072023-09-22T23:33:50.657Z 12 240-259 158 158 158 253 253 253 F F F
212082023-09-22T23:33:50.660Z 13 260-279 160 160 159 257 257 255 F F F <---
212092023-09-22T23:33:50.663Z 14 280-299 160 160 148 257 257 229 F F F <---
212102023-09-22T23:33:50.666Z 15 300-319 149 149 149 232 232 232 F F F
212112023-09-22T23:33:50.668Z 16 320-339 158 158 158 252 252 252 F F F
212122023-09-22T23:33:50.671Z 17 340-359 157 157 157 250 250 250 F F F
212132023-09-22T23:33:50.674Z 18 360-379 155 155 155 248 248 248 F F F
212142023-09-22T23:33:50.677Z 19 380-399 160 160 157 256 256 250 F F T <---
212152023-09-22T23:33:50.680Z 20 400-419 160 160 158 257 257 252 F F F <---
212162023-09-22T23:33:50.683Z 21 420-439 158 158 158 252 252 252 F F F
212172023-09-22T23:33:50.686Z 22 440-459 158 158 158 253 253 253 F F F
212182023-09-22T23:33:50.689Z 23 460-479 160 160 158 256 256 254 T T T <---
212192023-09-22T23:33:50.691Z 24 480-499 155 155 155 248 248 248 T T F <---
212202023-09-22T23:33:50.694Z 25 500-519 154 154 154 247 247 247 F F F
212212023-09-22T23:33:50.697Z 26 520-539 160 160 152 257 257 238 F F F <---
212222023-09-22T23:33:50.700Z 27 540-559 160 160 154 257 257 247 F F F <---
212232023-09-22T23:33:50.703Z 28 560-579 160 160 154 257 257 247 F F F <---
212242023-09-22T23:33:50.705Z 29 580-599 160 160 151 257 257 236 F F F <---
212252023-09-22T23:33:50.708ZMax gen: 160, Max flush: 257
212262023-09-22T23:33:50.711ZA Difference in extent metadata is expected here
212272023-09-22T23:33:50.714ZOn loop 80
212282023-09-22T23:33:50.716Z
212292023-09-22T23:33:50.719ZVerifying data now
212302023-09-22T23:33:56.814ZNo change after verify
212312023-09-22T23:33:56.817ZLoop: 80 Downstairs dump after verify (and repair):
212322023-09-22T23:33:56.832Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.82982014Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1826}
212332023-09-22T23:33:56.835Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.830197503Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1826}
212342023-09-22T23:33:56.838Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.830222496Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1826}
212352023-09-22T23:33:56.851Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.849188731Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1826}
212362023-09-22T23:33:56.854Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.849241488Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1826}
212372023-09-22T23:33:56.857Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.849262132Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1826}
212382023-09-22T23:33:56.862Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.860408867Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1826}
212392023-09-22T23:33:56.864Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.860461342Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1826}
212402023-09-22T23:33:56.867Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.860481698Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1826}
212412023-09-22T23:33:56.873ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
212422023-09-22T23:33:56.876Z 0 000-019 153 153 153 242 242 242 F F F
212432023-09-22T23:33:56.879Z 1 020-039 157 157 157 250 250 250 F F F
212442023-09-22T23:33:56.882Z 2 040-059 161 161 161 258 258 258 F F F
212452023-09-22T23:33:56.885Z 3 060-079 160 160 160 257 257 257 F F F
212462023-09-22T23:33:56.887Z 4 080-099 156 156 156 249 249 249 F F F
212472023-09-22T23:33:56.890Z 5 100-119 154 154 154 247 247 247 F F F
212482023-09-22T23:33:56.893Z 6 120-139 151 151 151 236 236 236 F F F
212492023-09-22T23:33:56.896Z 7 140-159 157 157 157 250 250 250 F F F
212502023-09-22T23:33:56.898Z 8 160-179 157 157 157 250 250 250 F F F
212512023-09-22T23:33:56.901Z 9 180-199 157 157 157 250 250 250 F F F
212522023-09-22T23:33:56.904Z 10 200-219 153 153 153 242 242 242 F F F
212532023-09-22T23:33:56.907Z 11 220-239 161 161 161 258 258 258 F F F
212542023-09-22T23:33:56.910Z 12 240-259 158 158 158 253 253 253 F F F
212552023-09-22T23:33:56.913Z 13 260-279 160 160 160 257 257 257 F F F
212562023-09-22T23:33:56.916Z 14 280-299 160 160 160 257 257 257 F F F
212572023-09-22T23:33:56.919Z 15 300-319 149 149 149 232 232 232 F F F
212582023-09-22T23:33:56.922Z 16 320-339 158 158 158 252 252 252 F F F
212592023-09-22T23:33:56.925Z 17 340-359 157 157 157 250 250 250 F F F
212602023-09-22T23:33:56.927Z 18 360-379 155 155 155 248 248 248 F F F
212612023-09-22T23:33:56.930Z 19 380-399 160 160 160 256 256 256 F F F
212622023-09-22T23:33:56.933Z 20 400-419 160 160 160 257 257 257 F F F
212632023-09-22T23:33:56.936Z 21 420-439 158 158 158 252 252 252 F F F
212642023-09-22T23:33:56.939Z 22 440-459 158 158 158 253 253 253 F F F
212652023-09-22T23:33:56.942Z 23 460-479 161 161 161 258 258 258 F F F
212662023-09-22T23:33:56.944Z 24 480-499 161 161 161 258 258 258 F F F
212672023-09-22T23:33:56.947Z 25 500-519 154 154 154 247 247 247 F F F
212682023-09-22T23:33:56.950Z 26 520-539 160 160 160 257 257 257 F F F
212692023-09-22T23:33:56.952Z 27 540-559 160 160 160 257 257 257 F F F
212702023-09-22T23:33:56.955Z 28 560-579 160 160 160 257 257 257 F F F
212712023-09-22T23:33:56.958Z 29 580-599 160 160 160 257 257 257 F F F
212722023-09-22T23:33:56.961ZMax gen: 161, Max flush: 258
212732023-09-22T23:33:56.964Z
212742023-09-22T23:33:56.967ZBegin loop 81
212752023-09-22T23:33:56.969ZDownstairs to restart: 1
212762023-09-22T23:33:56.972Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.909300085Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
212772023-09-22T23:33:56.975Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.909671381Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
212782023-09-22T23:33:56.978Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.909723216Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
212792023-09-22T23:33:56.981Z{"msg":"Crucible stats registered with UUID: ab8ea343-b502-4343-8718-a9472e59395a","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.909774718Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
212802023-09-22T23:33:56.984Z{"msg":"Crucible ab8ea343-b502-4343-8718-a9472e59395a has session id: d9bbe594-02e3-4e5a-a245-91db19b769fd","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.909821611Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
212812023-09-22T23:33:56.986ZCrucible runtime is spawned
212822023-09-22T23:33:56.989Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30The guest has requested activation with gen:162
212832023-09-22T23:33:56.992Z,"time":"2023-09-22T23:33:56.910068816Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828,"task":"up_listen"}
212842023-09-22T23:33:56.994Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.910197139Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
212852023-09-22T23:33:56.997Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.910275603Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
212862023-09-22T23:33:57.000Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.910340164Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828,"looper":"0"}
212872023-09-22T23:33:57.003Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.910597262Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828,"looper":"1"}
212882023-09-22T23:33:57.006Z{"msg":"[0] ab8ea343-b502-4343-8718-a9472e59395a looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.910664586Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828,"looper":"0"}
212892023-09-22T23:33:57.008Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.910725206Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828,"looper":"2"}
212902023-09-22T23:33:57.011Z{"msg":"[2] ab8ea343-b502-4343-8718-a9472e59395a looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.910919036Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828,"looper":"2"}
212912023-09-22T23:33:57.014Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.910991661Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
212922023-09-22T23:33:57.017Z{"msg":"ab8ea343-b502-4343-8718-a9472e59395a active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.911046618Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
212932023-09-22T23:33:57.020Z{"msg":"Set desired generation to :162","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.911099833Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
212942023-09-22T23:33:57.022Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.911391777Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
212952023-09-22T23:33:57.025Z{"msg":"[0] ab8ea343-b502-4343-8718-a9472e59395a (be233eea-55d0-494f-92f2-44d8a1c2cb38) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.911707617Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
212962023-09-22T23:33:57.028Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.91178647Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
212972023-09-22T23:33:57.031Z{"msg":"[0] client is_active_req TRUE, promote! session be233eea-55d0-494f-92f2-44d8a1c2cb38","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.911861851Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
212982023-09-22T23:33:57.034Z{"msg":"[2] ab8ea343-b502-4343-8718-a9472e59395a (be233eea-55d0-494f-92f2-44d8a1c2cb38) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.911992595Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
212992023-09-22T23:33:57.036Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.912047934Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213002023-09-22T23:33:57.039Z{"msg":"[2] client is_active_req TRUE, promote! session be233eea-55d0-494f-92f2-44d8a1c2cb38","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.912090329Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213012023-09-22T23:33:57.042Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.912457719Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213022023-09-22T23:33:57.045Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.912522796Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213032023-09-22T23:33:57.048Z{"msg":"ab8ea343-b502-4343-8718-a9472e59395a WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.912649005Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213042023-09-22T23:33:57.050Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.91279338Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213052023-09-22T23:33:57.053Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.912860628Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213062023-09-22T23:33:57.056Z{"msg":"ab8ea343-b502-4343-8718-a9472e59395a WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.912930691Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213072023-09-22T23:33:57.058Z{"msg":"[0] ab8ea343-b502-4343-8718-a9472e59395a (be233eea-55d0-494f-92f2-44d8a1c2cb38) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.914669289Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213082023-09-22T23:33:57.062Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.914742869Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213092023-09-22T23:33:57.064Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:56.914801205Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213102023-09-22T23:33:57.068Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.914879104Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213112023-09-22T23:33:57.071Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.914922164Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213122023-09-22T23:33:57.074Z{"msg":"ab8ea343-b502-4343-8718-a9472e59395a WaitQuorum New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.914956866Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213132023-09-22T23:33:57.077Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.914985892Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213142023-09-22T23:33:57.079Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.915041728Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213152023-09-22T23:33:57.082Z{"msg":"[2] ab8ea343-b502-4343-8718-a9472e59395a (be233eea-55d0-494f-92f2-44d8a1c2cb38) WaitQuorum New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.915106409Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213162023-09-22T23:33:57.085Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.915159677Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213172023-09-22T23:33:57.087Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:56.915214094Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213182023-09-22T23:33:57.090Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.915267264Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213192023-09-22T23:33:57.093Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.915320758Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213202023-09-22T23:33:57.096Z{"msg":"ab8ea343-b502-4343-8718-a9472e59395a WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.91537475Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213212023-09-22T23:33:57.098Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.91542843Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213222023-09-22T23:33:57.101Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:56.91548084Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213232023-09-22T23:33:57.914Z{"msg":"[1] ab8ea343-b502-4343-8718-a9472e59395a looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.912141548Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828,"looper":"1"}
213242023-09-22T23:33:57.917Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.912223711Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213252023-09-22T23:33:57.921Z{"msg":"[1] ab8ea343-b502-4343-8718-a9472e59395a (be233eea-55d0-494f-92f2-44d8a1c2cb38) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.913384647Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213262023-09-22T23:33:57.924Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.913441018Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213272023-09-22T23:33:57.927Z{"msg":"[1] client is_active_req TRUE, promote! session be233eea-55d0-494f-92f2-44d8a1c2cb38","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.913486279Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213282023-09-22T23:33:57.930Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.914063715Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213292023-09-22T23:33:57.933Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.914125517Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213302023-09-22T23:33:57.936Z{"msg":"ab8ea343-b502-4343-8718-a9472e59395a WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.914169441Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213312023-09-22T23:33:57.939Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.916329271Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213322023-09-22T23:33:57.942Z{"msg":"[1] ab8ea343-b502-4343-8718-a9472e59395a (be233eea-55d0-494f-92f2-44d8a1c2cb38) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.916563903Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213332023-09-22T23:33:57.945Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.916647652Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213342023-09-22T23:33:57.947Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:33:57.916716527Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213352023-09-22T23:33:57.950Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.916780473Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213362023-09-22T23:33:57.953Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.916847133Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213372023-09-22T23:33:57.956Z{"msg":"ab8ea343-b502-4343-8718-a9472e59395a WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.916911625Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213382023-09-22T23:33:57.959Z{"msg":"[0]R flush_numbers[0..12]: [242, 250, 258, 257, 249, 247, 236, 250, 250, 250, 242, 258]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.916993879Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213392023-09-22T23:33:57.961ZThe guest has finished waiting for activation with:162
213402023-09-22T23:33:57.964Z{"Wait for a query_work_queue command to finish before sending IO
213412023-09-22T23:33:57.967Zmsg":"[0]R generation[0..12]: [153, 157, 161, 160, 156, 154, 151, 157, 157, 157, 153, 161]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.917067224Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213422023-09-22T23:33:57.970Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.917164567Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213432023-09-22T23:33:57.973Z{"msg":"[1]R flush_numbers[0..12]: [242, 250, 258, 257, 249, 247, 236, 250, 250, 250, 242, 258]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.917233131Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213442023-09-22T23:33:57.976Z{"msg":"[1]R generation[0..12]: [153, 157, 161, 160, 156, 154, 151, 157, 157, 157, 153, 161]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.917301297Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213452023-09-22T23:33:57.979Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.917387738Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213462023-09-22T23:33:57.982Z{"msg":"[2]R flush_numbers[0..12]: [242, 250, 258, 257, 249, 247, 236, 250, 250, 250, 242, 258]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.917454705Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213472023-09-22T23:33:57.985Z{"msg":"[2]R generation[0..12]: [153, 157, 161, 160, 156, 154, 151, 157, 157, 157, 153, 161]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.91750011Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213482023-09-22T23:33:57.988Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.917549686Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213492023-09-22T23:33:57.991Z{"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.917592265Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213502023-09-22T23:33:57.993Z{"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.917654228Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213512023-09-22T23:33:57.997Z{"msg":"Next flush: 259","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.917696913Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213522023-09-22T23:33:57.999Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.917739422Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213532023-09-22T23:33:58.002Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.917793416Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213542023-09-22T23:33:58.005Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.917850905Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213552023-09-22T23:33:58.008Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.917904027Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213562023-09-22T23:33:58.011Z{"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
213572023-09-22T23:33:58.013Zmsg":"ab8ea343-b502-4343-8718-a9472e59395a is now active with session: be233eea-55d0-494f-92f2-44d8a1c2cb38","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.917965706Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213582023-09-22T23:33:58.016Z{"msg":"ab8ea343-b502-4343-8718-a9472e59395a Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.918008845Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213592023-09-22T23:33:58.019Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.918047673Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213602023-09-22T23:33:58.022Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.918084996Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213612023-09-22T23:33:58.025Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.918139474Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213622023-09-22T23:33:58.028Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.91818147Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213632023-09-22T23:33:58.031Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.918219314Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213642023-09-22T23:33:58.033Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.918257088Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213652023-09-22T23:33:58.036Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.918294564Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213662023-09-22T23:33:58.039Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.918331465Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213672023-09-22T23:33:58.042Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.918369087Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213682023-09-22T23:33:58.044Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.918406506Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213692023-09-22T23:33:58.047Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:57.918443652Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1828}
213702023-09-22T23:33:58.050ZLoading write count information from file "/tmp/test_repair_verify.data"
213712023-09-22T23:33:58.053ZRun Repair workload
213722023-09-22T23:33:58.055Z01/30 Write block 489 len 40960 data: 23 20 22 19 18 16 18 18 18 12
213732023-09-22T23:33:58.058Z02/30 Write block 108 len 36864 data: 21 19 19 16 16 16 14 12 12
213742023-09-22T23:33:58.061Z03/30 Read block 175 len 28672
213752023-09-22T23:33:58.064Z04/30 Read block 107 len 32768
213762023-09-22T23:33:58.067Z05/30 Read block 320 len 16384
213772023-09-22T23:33:58.069Z06/30 Flush
213782023-09-22T23:33:58.073Z07/30 Write block 12 len 32768 data: 10 12 10 12 11 11 11 10
213792023-09-22T23:33:58.076Z08/30 Read block 480 len 8192
213802023-09-22T23:33:58.078Z09/30 Read block 577 len 8192
213812023-09-22T23:33:58.081Z10/30 Write block 580 len 40960 data: 11 10 10 9 9 11 9 10 11 12
213822023-09-22T23:33:58.084Z11/30 Read block 228 len 12288
213832023-09-22T23:33:58.087Z12/30 Read block 583 len 28672
213842023-09-22T23:33:58.090Z13/30 Write block 91 len 36864 data: 14 14 14 13 15 15 13 16 14
213852023-09-22T23:33:58.093Z14/30 Write block 147 len 32768 data: 9 12 14 14 13 12 9 11
213862023-09-22T23:33:58.096Z15/30 Read block 288 len 4096
213872023-09-22T23:33:58.099Z16/30 Read block 338 len 28672
213882023-09-22T23:33:58.102Z17/30 Write block 120 len 8192 data: 11 11
213892023-09-22T23:33:58.105Z18/30 Read block 65 len 12288
213902023-09-22T23:33:58.107Z19/30 Read block 422 len 24576
213912023-09-22T23:33:58.110Z20/30 Write block 24 len 32768 data: 8 9 10 11 8 6 7 9
213922023-09-22T23:33:58.113Z21/30 Write block 488 len 8192 data: 22 24
213932023-09-22T23:33:58.116Z22/30 Read block 142 len 24576
213942023-09-22T23:33:58.119Z23/30 Write block 342 len 36864 data: 12 10 10 13 14 12 12 11 11
213952023-09-22T23:33:58.121Z24/30 Write block 523 len 20480 data: 12 11 14 13 14
213962023-09-22T23:33:58.124Z25/30 Write block 263 len 40960 data: 16 15 13 13 16 13 15 15 18 19
213972023-09-22T23:33:58.127Z26/30 Write block 107 len 32768 data: 21 22 20 20 17 17 17 15
213982023-09-22T23:33:58.130Z27/30 Write block 407 len 20480 data: 11 9 11 10 14
213992023-09-22T23:33:58.132Z28/30 Read block 509 len 12288
214002023-09-22T23:33:58.135Z29/30 Read block 468 len 16384
214012023-09-22T23:33:58.138Z30/30 Write block 265 len 36864 data: 14 14 17 14 16 16 19 20 18
214022023-09-22T23:33:58.141Z----------------------------------------------------------------
214032023-09-22T23:33:58.144Z Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:30
214042023-09-22T23:33:58.147ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
214052023-09-22T23:33:58.150Z 1 Acked 1000 Write 10 Done Done Done false
214062023-09-22T23:33:58.153Z 2 Acked 1001 Write 9 Done Sent Done false
214072023-09-22T23:33:58.156Z 3 Acked 1002 Read 7 Done Sent Done false
214082023-09-22T23:33:58.159Z 4 Acked 1003 Read 8 Done Sent Done false
214092023-09-22T23:33:58.161Z 5 Acked 1004 Read 4 Done Sent Done false
214102023-09-22T23:33:58.164Z 6 Acked 1005 Flush 0 Done Sent Done false
214112023-09-22T23:33:58.167Z 7 Acked 1006 Write 8 Done Sent Done false
214122023-09-22T23:33:58.170Z 8 Acked 1007 Read 2 Done Sent Done false
214132023-09-22T23:33:58.173Z 9 Acked 1008 Read 2 Done Sent Done false
214142023-09-22T23:33:58.175Z 10 Acked 1009 Write 10 Done Sent Done false
214152023-09-22T23:33:58.178Z 11 Acked 1010 Read 3 Done Sent Done false
214162023-09-22T23:33:58.181Z 12 Acked 1011 Read 7 Done Sent Done false
214172023-09-22T23:33:58.184Z 13 Acked 1012 Write 9 Done Sent Done false
214182023-09-22T23:33:58.186Z 14 Acked 1013 Write 8 Done Sent Done false
214192023-09-22T23:33:58.189Z 15 Acked 1014 Read 1 Done Sent Done false
214202023-09-22T23:33:58.192Z 16 Acked 1015 Read 7 Done Sent Done false
214212023-09-22T23:33:58.195Z 17 Acked 1016 Write 2 Done Sent Done false
214222023-09-22T23:33:58.198Z 18 Acked 1017 Read 3 Done Sent Done false
214232023-09-22T23:33:58.201Z 19 Acked 1018 Read 6 Done Sent Done false
214242023-09-22T23:33:58.204Z 20 Acked 1019 Write 8 Done Sent Done false
214252023-09-22T23:33:58.207Z 21 Acked 1020 Write 2 Done Sent Done false
214262023-09-22T23:33:58.210Z 22 Acked 1021 Read 6 Done Sent Done false
214272023-09-22T23:33:58.212Z 23 Acked 1022 Write 9 Done Sent Done false
214282023-09-22T23:33:58.215Z 24 Acked 1023 Write 5 Done Sent Done false
214292023-09-22T23:33:58.218Z 25 Acked 1024 Write 10 Done Sent Done false
214302023-09-22T23:33:58.220Z 26 Acked 1025 Write 8 Done Sent Done false
214312023-09-22T23:33:58.223Z 27 Acked 1026 Write 5 Done Sent Done false
214322023-09-22T23:33:58.226Z 28 Acked 1027 Read 3 Done Sent Done false
214332023-09-22T23:33:58.229Z 29 Acked 1028 Read 4 Done Sent Done false
214342023-09-22T23:33:58.232Z 30 Acked 1029 Write 9 Sent Sent Sent false
214352023-09-22T23:33:58.235Z STATES DS:0 DS:1 DS:2 TOTAL
214362023-09-22T23:33:58.237Z New 0 0 0 0
214372023-09-22T23:33:58.240Z Sent 1 29 1 31
214382023-09-22T23:33:58.243Z Done 29 1 29 59
214392023-09-22T23:33:58.246Z Skipped 0 0 0 0
214402023-09-22T23:33:58.249Z Error 0 0 0 0
214412023-09-22T23:33:58.252ZLast Flush: 1005 0 1005
214422023-09-22T23:33:58.254ZDownstairs last five completed:
214432023-09-22T23:33:58.257ZUpstairs last five completed: 30 29 28 27 26 25
214442023-09-22T23:33:58.260ZWrote out file "/tmp/test_repair_verify.data"
214452023-09-22T23:33:58.263Z
214462023-09-22T23:33:58.266ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
214472023-09-22T23:33:58.268Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:58.046973316Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1829}
214482023-09-22T23:33:58.271Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:58.047341843Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1829}
214492023-09-22T23:33:58.274Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:58.047364798Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1829}
214502023-09-22T23:33:58.277Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:58.082967122Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1829}
214512023-09-22T23:33:58.280Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:58.083017909Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1829}
214522023-09-22T23:33:58.283Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:58.083039483Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1829}
214532023-09-22T23:33:58.285Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:58.103208941Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1829}
214542023-09-22T23:33:58.288Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:58.103324407Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1829}
214552023-09-22T23:33:58.291Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:33:58.103347277Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1829}
214562023-09-22T23:33:58.294ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
214572023-09-22T23:33:58.296Z 0 000-019 153 153 153 242 242 242 T F T <---
214582023-09-22T23:33:58.299Z 1 020-039 157 157 157 250 250 250 T F T <---
214592023-09-22T23:33:58.302Z 2 040-059 161 161 161 258 258 258 F F F
214602023-09-22T23:33:58.305Z 3 060-079 160 160 160 257 257 257 F F F
214612023-09-22T23:33:58.308Z 4 080-099 156 156 156 249 249 249 T F T <---
214622023-09-22T23:33:58.311Z 5 100-119 162 154 162 259 247 259 T F T <---
214632023-09-22T23:33:58.314Z 6 120-139 151 151 151 236 236 236 T F T <---
214642023-09-22T23:33:58.317Z 7 140-159 157 157 157 250 250 250 T F T <---
214652023-09-22T23:33:58.319ZError: Difference in extent metadata found!
214662023-09-22T23:33:58.322Z 8 160-179 157 157 157 250 250 250 F F F
214672023-09-22T23:33:58.325Z 9 180-199 157 157 157 250 250 250 F F F
214682023-09-22T23:33:58.328Z 10 200-219 153 153 153 242 242 242 F F F
214692023-09-22T23:33:58.330Z 11 220-239 161 161 161 258 258 258 F F F
214702023-09-22T23:33:58.333Z 12 240-259 158 158 158 253 253 253 F F F
214712023-09-22T23:33:58.336Z 13 260-279 160 160 160 257 257 257 T F T <---
214722023-09-22T23:33:58.338Z 14 280-299 160 160 160 257 257 257 F F F
214732023-09-22T23:33:58.341Z 15 300-319 149 149 149 232 232 232 F F F
214742023-09-22T23:33:58.344Z 16 320-339 158 158 158 252 252 252 F F F
214752023-09-22T23:33:58.347Z 17 340-359 157 157 157 250 250 250 T F T <---
214762023-09-22T23:33:58.350Z 18 360-379 155 155 155 248 248 248 F F F
214772023-09-22T23:33:58.353Z 19 380-399 160 160 160 256 256 256 F F F
214782023-09-22T23:33:58.355Z 20 400-419 160 160 160 257 257 257 T F T <---
214792023-09-22T23:33:58.359Z 21 420-439 158 158 158 252 252 252 F F F
214802023-09-22T23:33:58.361Z 22 440-459 158 158 158 253 253 253 F F F
214812023-09-22T23:33:58.364Z 23 460-479 161 161 161 258 258 258 F F F
214822023-09-22T23:33:58.367Z 24 480-499 162 161 162 259 258 259 T T T <---
214832023-09-22T23:33:58.370Z 25 500-519 154 154 154 247 247 247 F F F
214842023-09-22T23:33:58.373Z 26 520-539 160 160 160 257 257 257 T F T <---
214852023-09-22T23:33:58.376Z 27 540-559 160 160 160 257 257 257 F F F
214862023-09-22T23:33:58.378Z 28 560-579 160 160 160 257 257 257 F F F
214872023-09-22T23:33:58.382Z 29 580-599 160 160 160 257 257 257 T F T <---
214882023-09-22T23:33:58.384ZMax gen: 162, Max flush: 259
214892023-09-22T23:33:58.387ZA Difference in extent metadata is expected here
214902023-09-22T23:33:58.390ZOn loop 81
214912023-09-22T23:33:58.392Z
214922023-09-22T23:33:58.395ZVerifying data now
214932023-09-22T23:34:04.482ZNo change after verify
214942023-09-22T23:34:04.485ZLoop: 81 Downstairs dump after verify (and repair):
214952023-09-22T23:34:04.500Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.498282979Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1834}
214962023-09-22T23:34:04.504Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.498681455Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1834}
214972023-09-22T23:34:04.506Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.498706553Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1834}
214982023-09-22T23:34:04.519Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.517127446Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1834}
214992023-09-22T23:34:04.521Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.517176487Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1834}
215002023-09-22T23:34:04.524Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.517196533Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1834}
215012023-09-22T23:34:04.529Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.527975917Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1834}
215022023-09-22T23:34:04.532Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.528021959Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1834}
215032023-09-22T23:34:04.535Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.528041914Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1834}
215042023-09-22T23:34:04.540ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
215052023-09-22T23:34:04.543Z 0 000-019 163 163 163 260 260 260 F F F
215062023-09-22T23:34:04.546Z 1 020-039 163 163 163 260 260 260 F F F
215072023-09-22T23:34:04.549Z 2 040-059 161 161 161 258 258 258 F F F
215082023-09-22T23:34:04.552Z 3 060-079 160 160 160 257 257 257 F F F
215092023-09-22T23:34:04.555Z 4 080-099 163 163 163 260 260 260 F F F
215102023-09-22T23:34:04.559Z 5 100-119 163 163 163 260 260 260 F F F
215112023-09-22T23:34:04.562Z 6 120-139 163 163 163 260 260 260 F F F
215122023-09-22T23:34:04.565Z 7 140-159 163 163 163 260 260 260 F F F
215132023-09-22T23:34:04.567Z 8 160-179 157 157 157 250 250 250 F F F
215142023-09-22T23:34:04.571Z 9 180-199 157 157 157 250 250 250 F F F
215152023-09-22T23:34:04.573Z 10 200-219 153 153 153 242 242 242 F F F
215162023-09-22T23:34:04.577Z 11 220-239 161 161 161 258 258 258 F F F
215172023-09-22T23:34:04.580Z 12 240-259 158 158 158 253 253 253 F F F
215182023-09-22T23:34:04.583Z 13 260-279 163 163 163 260 260 260 F F F
215192023-09-22T23:34:04.586Z 14 280-299 160 160 160 257 257 257 F F F
215202023-09-22T23:34:04.589Z 15 300-319 149 149 149 232 232 232 F F F
215212023-09-22T23:34:04.592Z 16 320-339 158 158 158 252 252 252 F F F
215222023-09-22T23:34:04.595Z 17 340-359 163 163 163 260 260 260 F F F
215232023-09-22T23:34:04.598Z 18 360-379 155 155 155 248 248 248 F F F
215242023-09-22T23:34:04.601Z 19 380-399 160 160 160 256 256 256 F F F
215252023-09-22T23:34:04.604Z 20 400-419 163 163 163 260 260 260 F F F
215262023-09-22T23:34:04.607Z 21 420-439 158 158 158 252 252 252 F F F
215272023-09-22T23:34:04.610Z 22 440-459 158 158 158 253 253 253 F F F
215282023-09-22T23:34:04.613Z 23 460-479 161 161 161 258 258 258 F F F
215292023-09-22T23:34:04.616Z 24 480-499 163 163 163 260 260 260 F F F
215302023-09-22T23:34:04.619Z 25 500-519 154 154 154 247 247 247 F F F
215312023-09-22T23:34:04.622Z 26 520-539 163 163 163 260 260 260 F F F
215322023-09-22T23:34:04.625Z 27 540-559 160 160 160 257 257 257 F F F
215332023-09-22T23:34:04.628Z 28 560-579 160 160 160 257 257 257 F F F
215342023-09-22T23:34:04.631Z 29 580-599 163 163 163 260 260 260 F F F
215352023-09-22T23:34:04.636ZMax gen: 163, Max flush: 260
215362023-09-22T23:34:04.641Z
215372023-09-22T23:34:04.645ZBegin loop 82
215382023-09-22T23:34:04.648ZDownstairs to restart: 1
215392023-09-22T23:34:04.651Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.575373967Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215402023-09-22T23:34:04.654Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.575827585Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215412023-09-22T23:34:04.657Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.575891715Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215422023-09-22T23:34:04.661Z{"msg":"Crucible stats registered with UUID: afce43a5-a7f4-4f6c-9bae-dffcb14289a1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.575934551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215432023-09-22T23:34:04.664Z{"msg":"Crucible afce43a5-a7f4-4f6c-9bae-dffcb14289a1 has session id: a5af68ab-7d42-4a09-8350-1cd248aae96b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.575993044Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215442023-09-22T23:34:04.667ZCrucible runtime is spawned
215452023-09-22T23:34:04.670ZThe guest has requested activation with gen:164
215462023-09-22T23:34:04.673Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.576380457Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836,"task":"up_listen"}
215472023-09-22T23:34:04.676Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.576436295Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215482023-09-22T23:34:04.679Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.576477308Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836,"looper":"1"}
215492023-09-22T23:34:04.682Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.57652714Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215502023-09-22T23:34:04.685Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.576663549Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836,"looper":"0"}
215512023-09-22T23:34:04.688Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.576876601Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836,"looper":"2"}
215522023-09-22T23:34:04.691Z{"msg":"[0] afce43a5-a7f4-4f6c-9bae-dffcb14289a1 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.576940334Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836,"looper":"0"}
215532023-09-22T23:34:04.694Z{"msg":"[2] afce43a5-a7f4-4f6c-9bae-dffcb14289a1 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.577060632Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836,"looper":"2"}
215542023-09-22T23:34:04.697Z{"msg":"afce43a5-a7f4-4f6c-9bae-dffcb14289a1 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.577382671Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215552023-09-22T23:34:04.700Z{"msg":"Set desired generation to :164","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.577439236Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215562023-09-22T23:34:04.703Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.577495282Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215572023-09-22T23:34:04.706Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.577830094Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215582023-09-22T23:34:04.709Z{"msg":"[0] afce43a5-a7f4-4f6c-9bae-dffcb14289a1 (d9580547-5e61-4809-a2bc-5d06225b48da) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.578154632Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215592023-09-22T23:34:04.713Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.578215066Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215602023-09-22T23:34:04.715Z{"msg":"[0] client is_active_req TRUE, promote! session d9580547-5e61-4809-a2bc-5d06225b48da","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.578258652Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215612023-09-22T23:34:04.718Z{"msg":"[2] afce43a5-a7f4-4f6c-9bae-dffcb14289a1 (d9580547-5e61-4809-a2bc-5d06225b48da) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.578376585Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215622023-09-22T23:34:04.721Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.578440997Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215632023-09-22T23:34:04.725Z{"msg":"[2] client is_active_req TRUE, promote! session d9580547-5e61-4809-a2bc-5d06225b48da","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.578517979Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215642023-09-22T23:34:04.728Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.578912408Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215652023-09-22T23:34:04.751Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.578972321Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215662023-09-22T23:34:04.754Z{"msg":"afce43a5-a7f4-4f6c-9bae-dffcb14289a1 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.579014389Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215672023-09-22T23:34:04.757Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.579163575Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215682023-09-22T23:34:04.760Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.579231221Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215692023-09-22T23:34:04.763Z{"msg":"afce43a5-a7f4-4f6c-9bae-dffcb14289a1 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.579300479Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215702023-09-22T23:34:04.766Z{"msg":"[0] afce43a5-a7f4-4f6c-9bae-dffcb14289a1 (d9580547-5e61-4809-a2bc-5d06225b48da) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.581305137Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215712023-09-22T23:34:04.769Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.581394434Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215722023-09-22T23:34:04.772Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:34:04.581470546Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215732023-09-22T23:34:04.774Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.58154745Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215742023-09-22T23:34:04.777Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.581622446Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215752023-09-22T23:34:04.780Z{"msg":"afce43a5-a7f4-4f6c-9bae-dffcb14289a1 WaitQuorum New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.581696766Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215762023-09-22T23:34:04.783Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.581772411Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215772023-09-22T23:34:04.786Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.581829306Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215782023-09-22T23:34:04.789Z{"msg":"[2] afce43a5-a7f4-4f6c-9bae-dffcb14289a1 (d9580547-5e61-4809-a2bc-5d06225b48da) WaitQuorum New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.581874049Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215792023-09-22T23:34:04.792Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.581913881Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215802023-09-22T23:34:04.795Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:34:04.581953071Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215812023-09-22T23:34:04.797Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.581991698Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215822023-09-22T23:34:04.801Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.582030891Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215832023-09-22T23:34:04.804Z{"msg":"afce43a5-a7f4-4f6c-9bae-dffcb14289a1 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.582070243Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215842023-09-22T23:34:04.807Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.582110469Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215852023-09-22T23:34:04.810Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:04.58213093Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215862023-09-22T23:34:05.581Z{"msg":"[1] afce43a5-a7f4-4f6c-9bae-dffcb14289a1 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.578647505Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836,"looper":"1"}
215872023-09-22T23:34:05.585Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.578729724Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215882023-09-22T23:34:05.588Z{"msg":"[1] afce43a5-a7f4-4f6c-9bae-dffcb14289a1 (d9580547-5e61-4809-a2bc-5d06225b48da) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.579959383Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215892023-09-22T23:34:05.591Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.580017608Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215902023-09-22T23:34:05.594Z{"msg":"[1] client is_active_req TRUE, promote! session d9580547-5e61-4809-a2bc-5d06225b48da","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.58006196Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215912023-09-22T23:34:05.597Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.580651012Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215922023-09-22T23:34:05.600Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.580711275Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215932023-09-22T23:34:05.603Z{"msg":"afce43a5-a7f4-4f6c-9bae-dffcb14289a1 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.580754793Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215942023-09-22T23:34:05.606Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.582947609Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215952023-09-22T23:34:05.610Z{"msg":"[1] afce43a5-a7f4-4f6c-9bae-dffcb14289a1 (d9580547-5e61-4809-a2bc-5d06225b48da) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.583159011Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215962023-09-22T23:34:05.614Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.58324207Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215972023-09-22T23:34:05.617Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:34:05.583295608Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215982023-09-22T23:34:05.620Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.583337005Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
215992023-09-22T23:34:05.623Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.583376749Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216002023-09-22T23:34:05.626Z{"msg":"afce43a5-a7f4-4f6c-9bae-dffcb14289a1 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.583418157Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216012023-09-22T23:34:05.629Z{"msg":"[0]R flush_numbers[0..12]: [260, 260, 258, 257, 260, 260, 260, 260, 250, 250, 242, 258]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.58346533Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216022023-09-22T23:34:05.632Z{The guest has finished waiting for activation with:164
216032023-09-22T23:34:05.634Z"msg":"[0]R generation[0..12]: [163, 163, 161, 160, 163, 163, 163, 163, 157, 157, 153, 161]","v":0,"name":"Wait for a query_work_queue command to finish before sending IO
216042023-09-22T23:34:05.637Zcrucible","level":30,"time":"2023-09-22T23:34:05.583521437Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216052023-09-22T23:34:05.640Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.583567557Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216062023-09-22T23:34:05.643Z{"msg":"[1]R flush_numbers[0..12]: [260, 260, 258, 257, 260, 260, 260, 260, 250, 250, 242, 258]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.583622407Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216072023-09-22T23:34:05.646Z{"msg":"[1]R generation[0..12]: [163, 163, 161, 160, 163, 163, 163, 163, 157, 157, 153, 161]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.583681915Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216082023-09-22T23:34:05.649Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.583739818Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216092023-09-22T23:34:05.652Z{"msg":"[2]R flush_numbers[0..12]: [260, 260, 258, 257, 260, 260, 260, 260, 250, 250, 242, 258]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.583781828Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216102023-09-22T23:34:05.655Z{"msg":"[2]R generation[0..12]: [163, 163, 161, 160, 163, 163, 163, 163, 157, 157, 153, 161]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.583822Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216112023-09-22T23:34:05.658Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.583845388Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216122023-09-22T23:34:05.661Z{"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.583882237Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216132023-09-22T23:34:05.664Z{"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.583902687Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216142023-09-22T23:34:05.666Z{"msg":"Next flush: 261","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.583946335Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216152023-09-22T23:34:05.669Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.583995839Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216162023-09-22T23:34:05.672Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.58401897Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216172023-09-22T23:34:05.675Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.584057105Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216182023-09-22T23:34:05.678Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.584096433Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216192023-09-22T23:34:05.681Z{"msg":"afce43a5-a7f4-4f6c-9bae-dffcb14289a1 is now active with session: d9580547-5e61-4809-a2bc-5d06225b48da","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.584137146Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216202023-09-22T23:34:05.684Z{"msg":"afce43a5-a7f4-4f6c-9bae-dffcb14289a1 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.584157808Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216212023-09-22T23:34:05.687Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.584177258Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216222023-09-22T23:34:05.690Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.584213559Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216232023-09-22T23:34:05.693Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.58425155Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216242023-09-22T23:34:05.696Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.584271212Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216252023-09-22T23:34:05.699Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.584289159Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216262023-09-22T23:34:05.702Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.584325955Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216272023-09-22T23:34:05.705ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
216282023-09-22T23:34:05.708Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.584369654Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216292023-09-22T23:34:05.710Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.584408548Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216302023-09-22T23:34:05.713Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.58443051Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216312023-09-22T23:34:05.716Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.58446713Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216322023-09-22T23:34:05.719Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.584488025Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1836}
216332023-09-22T23:34:05.722ZLoading write count information from file "/tmp/test_repair_verify.data"
216342023-09-22T23:34:05.725ZRun Repair workload
216352023-09-22T23:34:05.728Z01/30 Write block 71 len 36864 data: 15 15 12 13 14 13 14 14 18
216362023-09-22T23:34:05.731Z02/30 Read block 469 len 24576
216372023-09-22T23:34:05.734Z03/30 Read block 530 len 28672
216382023-09-22T23:34:05.737Z04/30 Flush
216392023-09-22T23:34:05.740Z05/30 Write block 222 len 4096 data: 9
216402023-09-22T23:34:05.743Z06/30 Read block 436 len 4096
216412023-09-22T23:34:05.746Z07/30 Write block 590 len 12288 data: 12 14 16
216422023-09-22T23:34:05.749Z08/30 Flush
216432023-09-22T23:34:05.772Z09/30 Write block 283 len 20480 data: 14 13 13 13 13
216442023-09-22T23:34:05.775Z10/30 Write block 484 len 28672 data: 17 18 21 18 23 25 21
216452023-09-22T23:34:05.778Z11/30 Write block 72 len 8192 data: 16 13
216462023-09-22T23:34:05.781Z12/30 Read block 226 len 36864
216472023-09-22T23:34:05.784Z13/30 Write block 137 len 28672 data: 12 12 13 13 12 13 12
216482023-09-22T23:34:05.787Z14/30 Write block 153 len 24576 data: 10 12 13 12 13 15
216492023-09-22T23:34:05.790Z15/30 Write block 182 len 28672 data: 13 10 12 11 13 13 11
216502023-09-22T23:34:05.793Z16/30 Write block 301 len 40960 data: 12 14 15 13 13 13 13 11 10 7
216512023-09-22T23:34:05.796Z17/30 Read block 200 len 16384
216522023-09-22T23:34:05.799Z18/30 Read block 106 len 40960
216532023-09-22T23:34:05.802Z19/30 Read block 448 len 24576
216542023-09-22T23:34:05.804Z20/30 Write block 261 len 16384 data: 14 15 17 16
216552023-09-22T23:34:05.808Z21/30 Write block 145 len 8192 data: 9 9
216562023-09-22T23:34:05.811Z22/30 Read block 380 len 40960
216572023-09-22T23:34:05.814Z23/30 Flush
216582023-09-22T23:34:05.817Z24/30 Write block 527 len 40960 data: 15 13 12 10 12 14 17 13 15 14
216592023-09-22T23:34:05.820Z25/30 Read block 336 len 32768
216602023-09-22T23:34:05.823Z26/30 Write block 306 len 24576 data: 14 14 12 11 8 7
216612023-09-22T23:34:05.826Z27/30 Write block 426 len 12288 data: 11 13 14
216622023-09-22T23:34:05.829Z28/30 Read block 327 len 12288
216632023-09-22T23:34:05.832Z29/30 Read block 495 len 8192
216642023-09-22T23:34:05.835Z30/30 Write block 450 len 4096 data: 14
216652023-09-22T23:34:05.838Z----------------------------------------------------------------
216662023-09-22T23:34:05.841Z Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:22
216672023-09-22T23:34:05.844ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
216682023-09-22T23:34:05.847Z 9 Acked 1008 Write 5 Done Done Done false
216692023-09-22T23:34:05.850Z 10 Acked 1009 Write 7 Done Done Done false
216702023-09-22T23:34:05.853Z 11 Acked 1010 Write 2 Done Done Done false
216712023-09-22T23:34:05.856Z 12 Acked 1011 Read 9 Done Done Done false
216722023-09-22T23:34:05.859Z 13 Acked 1012 Write 7 Done Done Done false
216732023-09-22T23:34:05.862Z 14 Acked 1013 Write 6 Done Done Done false
216742023-09-22T23:34:05.865Z 15 Acked 1014 Write 7 Done Done Done false
216752023-09-22T23:34:05.869Z 16 Acked 1015 Write 10 Done Done Done false
216762023-09-22T23:34:05.872Z 17 Acked 1016 Read 4 Done Done Done false
216772023-09-22T23:34:05.875Z 18 Acked 1017 Read 10 Done Done Done false
216782023-09-22T23:34:05.878Z 19 Acked 1018 Read 6 Done Done Done false
216792023-09-22T23:34:05.881Z 20 Acked 1019 Write 4 Done Done Done false
216802023-09-22T23:34:05.884Z 21 Acked 1020 Write 2 Done Sent Done false
216812023-09-22T23:34:05.887Z 22 Acked 1021 Read 10 Done Sent Done false
216822023-09-22T23:34:05.890Z 23 Acked 1022 Flush 0 Done Sent Done false
216832023-09-22T23:34:05.893Z 24 Acked 1023 Write 10 Done Sent Done false
216842023-09-22T23:34:05.895Z 25 Acked 1024 Read 8 Done Sent Done false
216852023-09-22T23:34:05.898Z 26 Acked 1025 Write 6 Done Sent Done false
216862023-09-22T23:34:05.901Z 27 Acked 1026 Write 3 Done Sent Done false
216872023-09-22T23:34:05.904Z 28 Acked 1027 Read 3 Done Sent Done false
216882023-09-22T23:34:05.907Z 29 Acked 1028 Read 2 Done Sent Done false
216892023-09-22T23:34:05.910Z 30 Acked 1029 Write 1 Sent Sent Sent false
216902023-09-22T23:34:05.912Z STATES DS:0 DS:1 DS:2 TOTAL
216912023-09-22T23:34:05.915Z New 0 0 0 0
216922023-09-22T23:34:05.918Z Sent 1 10 1 12
216932023-09-22T23:34:05.921Z Done 21 12 21 54
216942023-09-22T23:34:05.923Z Skipped 0 0 0 0
216952023-09-22T23:34:05.926Z Error 0 0 0 0
216962023-09-22T23:34:05.929ZLast Flush: 1022 1007 1022
216972023-09-22T23:34:05.932ZDownstairs last five completed: 1007 1006 1005 1004 1003 1002
216982023-09-22T23:34:05.934ZUpstairs last five completed: 30 29 28 27 26 25
216992023-09-22T23:34:05.937ZWrote out file "/tmp/test_repair_verify.data"
217002023-09-22T23:34:05.940Z
217012023-09-22T23:34:05.943ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
217022023-09-22T23:34:05.946Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.716912396Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1837}
217032023-09-22T23:34:05.949Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.717267213Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1837}
217042023-09-22T23:34:05.952Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.717290852Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1837}
217052023-09-22T23:34:05.955Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.742213629Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1837}
217062023-09-22T23:34:05.959Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.742267426Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1837}
217072023-09-22T23:34:05.962Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.742287882Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1837}
217082023-09-22T23:34:05.965Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.793907437Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1837}
217092023-09-22T23:34:05.968Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.793962573Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1837}
217102023-09-22T23:34:05.970Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:05.794001447Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1837}
217112023-09-22T23:34:05.973ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
217122023-09-22T23:34:05.976Z 0 000-019 163 163 163 260 260 260 F F F
217132023-09-22T23:34:05.979Z 1 020-039 163 163 163 260 260 260 F F F
217142023-09-22T23:34:05.982Z 2 040-059 161 161 161 258 258 258 F F F
217152023-09-22T23:34:05.985Z 3 060-079 164 164 164 263 261 263 F T F <---
217162023-09-22T23:34:05.988Z 4 080-099 163 163 163 260 260 260 F F F
217172023-09-22T23:34:05.991Z 5 100-119 163 163 163 260 260 260 F F F
217182023-09-22T23:34:05.995Z 6 120-139 164 163 164 263 260 263 F T F <---
217192023-09-22T23:34:05.998Z 7 140-159 164 163 164 263 260 263 F T F <---
217202023-09-22T23:34:06.001Z 8 160-179 157 157 157 250 250 250 F F F
217212023-09-22T23:34:06.005Z 9 180-199 164 157 164 263 250 263 F T F <---
217222023-09-22T23:34:06.008ZError: Difference in extent metadata found!
217232023-09-22T23:34:06.011Z 10 200-219 153 153 153 242 242 242 F F F
217242023-09-22T23:34:06.014Z 11 220-239 164 164 164 262 262 262 F F F
217252023-09-22T23:34:06.017Z 12 240-259 158 158 158 253 253 253 F F F
217262023-09-22T23:34:06.020Z 13 260-279 164 163 164 263 260 263 F T F <---
217272023-09-22T23:34:06.023Z 14 280-299 164 160 164 263 257 263 F T F <---
217282023-09-22T23:34:06.025Z 15 300-319 164 149 164 263 232 263 T T T <---
217292023-09-22T23:34:06.028Z 16 320-339 158 158 158 252 252 252 F F F
217302023-09-22T23:34:06.031Z 17 340-359 163 163 163 260 260 260 F F F
217312023-09-22T23:34:06.034Z 18 360-379 155 155 155 248 248 248 F F F
217322023-09-22T23:34:06.037Z 19 380-399 160 160 160 256 256 256 F F F
217332023-09-22T23:34:06.040Z 20 400-419 163 163 163 260 260 260 F F F
217342023-09-22T23:34:06.043Z 21 420-439 158 158 158 252 252 252 T F T <---
217352023-09-22T23:34:06.046Z 22 440-459 158 158 158 253 253 253 T F T <---
217362023-09-22T23:34:06.049Z 23 460-479 161 161 161 258 258 258 F F F
217372023-09-22T23:34:06.052Z 24 480-499 164 163 164 263 260 263 F T F <---
217382023-09-22T23:34:06.055Z 25 500-519 154 154 154 247 247 247 F F F
217392023-09-22T23:34:06.058Z 26 520-539 163 163 163 260 260 260 T F T <---
217402023-09-22T23:34:06.061Z 27 540-559 160 160 160 257 257 257 F F F
217412023-09-22T23:34:06.064Z 28 560-579 160 160 160 257 257 257 F F F
217422023-09-22T23:34:06.067Z 29 580-599 164 164 164 262 262 262 F F F
217432023-09-22T23:34:06.070ZMax gen: 164, Max flush: 263
217442023-09-22T23:34:06.072ZA Difference in extent metadata is expected here
217452023-09-22T23:34:06.076ZOn loop 82
217462023-09-22T23:34:06.079Z
217472023-09-22T23:34:06.083ZVerifying data now
217482023-09-22T23:34:12.060ZNo change after verify
217492023-09-22T23:34:12.063ZLoop: 82 Downstairs dump after verify (and repair):
217502023-09-22T23:34:12.078Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.076410503Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1842}
217512023-09-22T23:34:12.081Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.076738305Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1842}
217522023-09-22T23:34:12.084Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.076763255Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1842}
217532023-09-22T23:34:12.097Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.09526894Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1842}
217542023-09-22T23:34:12.099Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.095327698Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1842}
217552023-09-22T23:34:12.102Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.095348951Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1842}
217562023-09-22T23:34:12.108Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.106245606Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1842}
217572023-09-22T23:34:12.110Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.106298913Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1842}
217582023-09-22T23:34:12.113Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.106319989Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1842}
217592023-09-22T23:34:12.118ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
217602023-09-22T23:34:12.121Z 0 000-019 163 163 163 260 260 260 F F F
217612023-09-22T23:34:12.124Z 1 020-039 163 163 163 260 260 260 F F F
217622023-09-22T23:34:12.127Z 2 040-059 161 161 161 258 258 258 F F F
217632023-09-22T23:34:12.130Z 3 060-079 164 164 164 263 263 263 F F F
217642023-09-22T23:34:12.133Z 4 080-099 163 163 163 260 260 260 F F F
217652023-09-22T23:34:12.136Z 5 100-119 163 163 163 260 260 260 F F F
217662023-09-22T23:34:12.139Z 6 120-139 164 164 164 263 263 263 F F F
217672023-09-22T23:34:12.180Z 7 140-159 164 164 164 263 263 263 F F F
217682023-09-22T23:34:12.184Z 8 160-179 157 157 157 250 250 250 F F F
217692023-09-22T23:34:12.187Z 9 180-199 164 164 164 263 263 263 F F F
217702023-09-22T23:34:12.190Z 10 200-219 153 153 153 242 242 242 F F F
217712023-09-22T23:34:12.193Z 11 220-239 164 164 164 262 262 262 F F F
217722023-09-22T23:34:12.196Z 12 240-259 158 158 158 253 253 253 F F F
217732023-09-22T23:34:12.199Z 13 260-279 164 164 164 263 263 263 F F F
217742023-09-22T23:34:12.202Z 14 280-299 164 164 164 263 263 263 F F F
217752023-09-22T23:34:12.205Z 15 300-319 165 165 165 264 264 264 F F F
217762023-09-22T23:34:12.207Z 16 320-339 158 158 158 252 252 252 F F F
217772023-09-22T23:34:12.210Z 17 340-359 163 163 163 260 260 260 F F F
217782023-09-22T23:34:12.213Z 18 360-379 155 155 155 248 248 248 F F F
217792023-09-22T23:34:12.216Z 19 380-399 160 160 160 256 256 256 F F F
217802023-09-22T23:34:12.219Z 20 400-419 163 163 163 260 260 260 F F F
217812023-09-22T23:34:12.222Z 21 420-439 165 165 165 264 264 264 F F F
217822023-09-22T23:34:12.225Z 22 440-459 165 165 165 264 264 264 F F F
217832023-09-22T23:34:12.228Z 23 460-479 161 161 161 258 258 258 F F F
217842023-09-22T23:34:12.230Z 24 480-499 164 164 164 263 263 263 F F F
217852023-09-22T23:34:12.233Z 25 500-519 154 154 154 247 247 247 F F F
217862023-09-22T23:34:12.236Z 26 520-539 165 165 165 264 264 264 F F F
217872023-09-22T23:34:12.239Z 27 540-559 160 160 160 257 257 257 F F F
217882023-09-22T23:34:12.242Z 28 560-579 160 160 160 257 257 257 F F F
217892023-09-22T23:34:12.245Z 29 580-599 164 164 164 262 262 262 F F F
217902023-09-22T23:34:12.247ZMax gen: 165, Max flush: 264
217912023-09-22T23:34:12.250Z
217922023-09-22T23:34:12.253ZBegin loop 83
217932023-09-22T23:34:12.256ZDownstairs to restart: 1
217942023-09-22T23:34:12.259Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.155602252Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
217952023-09-22T23:34:12.262Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.15598884Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
217962023-09-22T23:34:12.265Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.156069842Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
217972023-09-22T23:34:12.268Z{"msg":"Crucible stats registered with UUID: 96bec209-ea1d-4f03-a265-fc8daf09084b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.156164393Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
217982023-09-22T23:34:12.271Z{"msg":"Crucible 96bec209-ea1d-4f03-a265-fc8daf09084b has session id: aa9eb1a9-fc1e-42be-a79a-e22490f4be5d","v":0,"name":"crucible","level":30Crucible runtime is spawned
217992023-09-22T23:34:12.274Z,"time":"2023-09-22T23:34:12.156303796Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218002023-09-22T23:34:12.277Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.156427528Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844The guest has requested activation with gen:166
218012023-09-22T23:34:12.280Z,"task":"up_listen"}
218022023-09-22T23:34:12.283Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.156474025Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218032023-09-22T23:34:12.285Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.156529799Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218042023-09-22T23:34:12.288Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.156571697Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844,"looper":"1"}
218052023-09-22T23:34:12.291Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.156797089Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844,"looper":"0"}
218062023-09-22T23:34:12.295Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.157033722Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844,"looper":"2"}
218072023-09-22T23:34:12.297Z{"msg":"[0] 96bec209-ea1d-4f03-a265-fc8daf09084b looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.157120887Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844,"looper":"0"}
218082023-09-22T23:34:12.300Z{"msg":"[2] 96bec209-ea1d-4f03-a265-fc8daf09084b looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.157363419Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844,"looper":"2"}
218092023-09-22T23:34:12.304Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.157464895Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218102023-09-22T23:34:12.306Z{"msg":"96bec209-ea1d-4f03-a265-fc8daf09084b active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.157551457Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218112023-09-22T23:34:12.309Z{"msg":"Set desired generation to :166","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.157635274Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218122023-09-22T23:34:12.312Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.157809387Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218132023-09-22T23:34:12.315Z{"msg":"[0] 96bec209-ea1d-4f03-a265-fc8daf09084b (2d4421df-f526-497d-92fb-c65d9f714b2f) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.158063642Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218142023-09-22T23:34:12.318Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.158127737Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218152023-09-22T23:34:12.321Z{"msg":"[0] client is_active_req TRUE, promote! session 2d4421df-f526-497d-92fb-c65d9f714b2f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.158171237Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218162023-09-22T23:34:12.324Z{"msg":"[2] 96bec209-ea1d-4f03-a265-fc8daf09084b (2d4421df-f526-497d-92fb-c65d9f714b2f) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.158293342Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218172023-09-22T23:34:12.326Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.158429318Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218182023-09-22T23:34:12.329Z{"msg":"[2] client is_active_req TRUE, promote! session 2d4421df-f526-497d-92fb-c65d9f714b2f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.158490597Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218192023-09-22T23:34:12.332Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.158815281Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218202023-09-22T23:34:12.335Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.158882148Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218212023-09-22T23:34:12.337Z{"msg":"96bec209-ea1d-4f03-a265-fc8daf09084b WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.158944598Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218222023-09-22T23:34:12.340Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.159097322Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218232023-09-22T23:34:12.343Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.159159355Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218242023-09-22T23:34:12.346Z{"msg":"96bec209-ea1d-4f03-a265-fc8daf09084b WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.159201092Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218252023-09-22T23:34:12.349Z{"msg":"[0] 96bec209-ea1d-4f03-a265-fc8daf09084b (2d4421df-f526-497d-92fb-c65d9f714b2f) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.161168748Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218262023-09-22T23:34:12.352Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.161295494Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218272023-09-22T23:34:12.355Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:34:12.161340024Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218282023-09-22T23:34:12.358Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.161378285Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218292023-09-22T23:34:12.362Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.161417209Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218302023-09-22T23:34:12.366Z{"msg":"96bec209-ea1d-4f03-a265-fc8daf09084b WaitQuorum New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.161455784Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218312023-09-22T23:34:12.369Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.161493895Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218322023-09-22T23:34:12.372Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.161531707Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218332023-09-22T23:34:12.375Z{"msg":"[2] 96bec209-ea1d-4f03-a265-fc8daf09084b (2d4421df-f526-497d-92fb-c65d9f714b2f) WaitQuorum New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.161572899Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218342023-09-22T23:34:12.378Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.161593185Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218352023-09-22T23:34:12.381Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:34:12.161610625Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218362023-09-22T23:34:12.384Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.161627864Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218372023-09-22T23:34:12.387Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.161645569Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218382023-09-22T23:34:12.389Z{"msg":"96bec209-ea1d-4f03-a265-fc8daf09084b WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.161663342Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218392023-09-22T23:34:12.392Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.161680565Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218402023-09-22T23:34:12.403Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:12.161697611Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218412023-09-22T23:34:13.160Z{"msg":"[1] 96bec209-ea1d-4f03-a265-fc8daf09084b looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.157652643Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844,"looper":"1"}
218422023-09-22T23:34:13.163Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.157770923Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218432023-09-22T23:34:13.166Z{"msg":"[1] 96bec209-ea1d-4f03-a265-fc8daf09084b (2d4421df-f526-497d-92fb-c65d9f714b2f) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.158977455Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218442023-09-22T23:34:13.170Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.159040579Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218452023-09-22T23:34:13.173Z{"msg":"[1] client is_active_req TRUE, promote! session 2d4421df-f526-497d-92fb-c65d9f714b2f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.1590852Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218462023-09-22T23:34:13.176Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.159736699Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218472023-09-22T23:34:13.180Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.159797844Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218482023-09-22T23:34:13.183Z{"msg":"96bec209-ea1d-4f03-a265-fc8daf09084b WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.159840925Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218492023-09-22T23:34:13.186Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.162295223Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218502023-09-22T23:34:13.189Z{"msg":"[1] 96bec209-ea1d-4f03-a265-fc8daf09084b (2d4421df-f526-497d-92fb-c65d9f714b2f) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.16240819Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218512023-09-22T23:34:13.193Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.16245813Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218522023-09-22T23:34:13.198Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:34:13.162499308Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218532023-09-22T23:34:13.201Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.162520115Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218542023-09-22T23:34:13.204Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.162557881Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218552023-09-22T23:34:13.207Z{"msg":"96bec209-ea1d-4f03-a265-fc8daf09084b WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.162598061Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218562023-09-22T23:34:13.210Z{"msg":"[0]R flush_numbers[0..12]: [260, 260, 258, 263, 260, 260, 263, 263, 250, 263, 242, 262]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.162638607Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218572023-09-22T23:34:13.214Z{"msg":"[0]R generation[0..12]: [163, 163, 161, 164, 163, 163, 164, 164, 157, 164, 153, 164]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.162678719Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218582023-09-22T23:34:13.217Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.162719328Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218592023-09-22T23:34:13.220Z{"msg":"[1]R flush_numbers[0..12]: [260, 260, 258, 263, 260, 260, 263, 263, 250, 263, 242, 262]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.162759202Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218602023-09-22T23:34:13.223Z{"msg":"[1]R generation[0..12]: [163, 163, 161, 164, 163, 163, 164, 164, 157, 164, 153, 164]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.162804083Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218612023-09-22T23:34:13.227Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:166
218622023-09-22T23:34:13.230ZWait for a query_work_queue command to finish before sending IO
218632023-09-22T23:34:13.234Z,"time":"2023-09-22T23:34:13.162871959Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218642023-09-22T23:34:13.237Z{"msg":"[2]R flush_numbers[0..12]: [260, 260, 258, 263, 260, 260, 263, 263, 250, 263, 242, 262]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.163045351Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218652023-09-22T23:34:13.240Z{"msg":"[2]R generation[0..12]: [163, 163, 161, 164, 163, 163, 164, 164, 157, 164, 153, 164]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.16310102Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218662023-09-22T23:34:13.243Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.16316116Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218672023-09-22T23:34:13.246Z{"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.163276437Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218682023-09-22T23:34:13.250Z{"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.163300951Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218692023-09-22T23:34:13.253Z{"msg":"Next flush: 265","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.163338219Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218702023-09-22T23:34:13.256Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.163385707Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218712023-09-22T23:34:13.260Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.163406381Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218722023-09-22T23:34:13.263Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.163443437Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218732023-09-22T23:34:13.270Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.163483957Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218742023-09-22T23:34:13.273Z{"msg":"96bec209-ea1d-4f03-a265-fc8daf09084b is now active with session: 2d4421df-f526-497d-92fb-c65d9f714b2f","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.163505439Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218752023-09-22T23:34:13.276Z{"msg":"96bec209-ea1d-4f03-a265-fc8daf09084b Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.16355102Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218762023-09-22T23:34:13.279Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.163599821Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218772023-09-22T23:34:13.282Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.163619351Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218782023-09-22T23:34:13.285Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.163636914Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218792023-09-22T23:34:13.288Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.16367334Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218802023-09-22T23:34:13.291Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.163716805Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218812023-09-22T23:34:13.294Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.163755474Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218822023-09-22T23:34:13.297Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.163774545Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218832023-09-22T23:34:13.300Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.163817882Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218842023-09-22T23:34:13.303Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.163855606Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218852023-09-22T23:34:13.306Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.163875116Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218862023-09-22T23:34:13.310Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.1639112Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1844}
218872023-09-22T23:34:13.313ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
218882023-09-22T23:34:13.316ZLoading write count information from file "/tmp/test_repair_verify.data"
218892023-09-22T23:34:13.319ZRun Repair workload
218902023-09-22T23:34:13.322Z01/30 Write block 247 len 28672 data: 9 9 10 8 7 11 10
218912023-09-22T23:34:13.325Z02/30 Write block 362 len 40960 data: 11 9 8 8 9 9 12 11 8 9
218922023-09-22T23:34:13.328Z03/30 Write block 581 len 32768 data: 11 11 10 10 12 10 11 12
218932023-09-22T23:34:13.331Z04/30 Write block 134 len 4096 data: 11
218942023-09-22T23:34:13.335Z05/30 Flush
218952023-09-22T23:34:13.337Z06/30 Write block 418 len 16384 data: 15 16 14 13
218962023-09-22T23:34:13.341Z07/30 Read block 51 len 20480
218972023-09-22T23:34:13.345Z08/30 Read block 161 len 8192
218982023-09-22T23:34:13.349Z09/30 Read block 495 len 40960
218992023-09-22T23:34:13.352Z10/30 Write block 84 len 28672 data: 11 9 10 14 11 11 11
219002023-09-22T23:34:13.355Z11/30 Flush
219012023-09-22T23:34:13.358Z12/30 Write block 203 len 28672 data: 17 17 21 19 19 22 18
219022023-09-22T23:34:13.361Z13/30 Write block 151 len 8192 data: 14 13
219032023-09-22T23:34:13.364Z14/30 Read block 499 len 12288
219042023-09-22T23:34:13.367Z15/30 Read block 552 len 24576
219052023-09-22T23:34:13.371Z16/30 Flush
219062023-09-22T23:34:13.374Z17/30 Write block 547 len 28672 data: 11 14 17 16 16 16 14
219072023-09-22T23:34:13.377Z18/30 Read block 6 len 20480
219082023-09-22T23:34:13.380Z19/30 Read block 484 len 4096
219092023-09-22T23:34:13.383Z20/30 Read block 408 len 8192
219102023-09-22T23:34:13.386Z21/30 Write block 378 len 32768 data: 12 11 10 11 10 9 9 9
219112023-09-22T23:34:13.390Z22/30 Write block 133 len 20480 data: 12 12 10 11 13
219122023-09-22T23:34:13.393Z23/30 Read block 461 len 16384
219132023-09-22T23:34:13.396Z24/30 Write block 559 len 36864 data: 10 13 11 11 11 13 14 14 16
219142023-09-22T23:34:13.399Z25/30 Read block 439 len 36864
219152023-09-22T23:34:13.402Z26/30 Flush
219162023-09-22T23:34:13.405Z27/30 Write block 499 len 20480 data: 12 12 11 12 11
219172023-09-22T23:34:13.409Z28/30 Write block 397 len 36864 data: 15 15 13 17 14 16 15 15 13
219182023-09-22T23:34:13.412Z29/30 Write block 523 len 32768 data: 13 12 15 14 16 14 13 11
219192023-09-22T23:34:13.415Z30/30 Write block 135 len 28672 data: 11 12 14 13 14 14 13
219202023-09-22T23:34:13.418Z----------------------------------------------------------------
219212023-09-22T23:34:13.421Z Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:25
219222023-09-22T23:34:13.424ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
219232023-09-22T23:34:13.430Z 6 Acked 1005 Write 4 Done Done Done false
219242023-09-22T23:34:13.434Z 7 Acked 1006 Read 5 Done Done Done false
219252023-09-22T23:34:13.438Z 8 Acked 1007 Read 2 Done Sent Done false
219262023-09-22T23:34:13.441Z 9 Acked 1008 Read 10 Done Sent Done false
219272023-09-22T23:34:13.445Z 10 Acked 1009 Write 7 Done Sent Done false
219282023-09-22T23:34:13.448Z 11 Acked 1010 Flush 0 Done Sent Done false
219292023-09-22T23:34:13.451Z 12 Acked 1011 Write 7 Done Sent Done false
219302023-09-22T23:34:13.454Z 13 Acked 1012 Write 2 Done Sent Done false
219312023-09-22T23:34:13.458Z 14 Acked 1013 Read 3 Done Sent Done false
219322023-09-22T23:34:13.461Z 15 Acked 1014 Read 6 Done Sent Done false
219332023-09-22T23:34:13.464Z 16 Acked 1015 Flush 0 Done Sent Done false
219342023-09-22T23:34:13.467Z 17 Acked 1016 Write 7 Done Sent Done false
219352023-09-22T23:34:13.470Z 18 Acked 1017 Read 5 Done Sent Done false
219362023-09-22T23:34:13.473Z 19 Acked 1018 Read 1 Done Sent Done false
219372023-09-22T23:34:13.477Z 20 Acked 1019 Read 2 Done Sent Done false
219382023-09-22T23:34:13.480Z 21 Acked 1020 Write 8 Done Sent Done false
219392023-09-22T23:34:13.483Z 22 Acked 1021 Write 5 Done Sent Done false
219402023-09-22T23:34:13.486Z 23 Acked 1022 Read 4 Done Sent Done false
219412023-09-22T23:34:13.491Z 24 Acked 1023 Write 9 Done Sent Done false
219422023-09-22T23:34:13.494Z 25 Acked 1024 Read 9 Done Sent Done false
219432023-09-22T23:34:13.498Z 26 Acked 1025 Flush 0 Done Sent Done false
219442023-09-22T23:34:13.501Z 27 Acked 1026 Write 5 Sent Sent Sent false
219452023-09-22T23:34:13.505Z 28 Acked 1027 Write 9 Sent Sent Sent false
219462023-09-22T23:34:13.508Z 29 Acked 1028 Write 8 Sent Sent Sent false
219472023-09-22T23:34:13.511Z 30 Acked 1029 Write 7 Sent Sent Sent false
219482023-09-22T23:34:13.514Z STATES DS:0 DS:1 DS:2 TOTAL
219492023-09-22T23:34:13.517Z New 0 0 0 0
219502023-09-22T23:34:13.520Z Sent 4 23 4 31
219512023-09-22T23:34:13.523Z Done 21 2 21 44
219522023-09-22T23:34:13.526Z Skipped 0 0 0 0
219532023-09-22T23:34:13.529Z Error 0 0 0 0
219542023-09-22T23:34:13.532ZLast Flush: 1025 1004 1025
219552023-09-22T23:34:13.535ZDownstairs last five completed: 1004 1003 1002 1001 1000
219562023-09-22T23:34:13.539ZUpstairs last five completed: 30 29 28 27 26 25
219572023-09-22T23:34:13.542ZWrote out file "/tmp/test_repair_verify.data"
219582023-09-22T23:34:13.545Z
219592023-09-22T23:34:13.548ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
219602023-09-22T23:34:13.551Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.293670855Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1845}
219612023-09-22T23:34:13.554Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.293930506Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1845}
219622023-09-22T23:34:13.557Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.293953724Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1845}
219632023-09-22T23:34:13.560Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.320403286Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1845}
219642023-09-22T23:34:13.563Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.320437322Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1845}
219652023-09-22T23:34:13.566Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.32045639Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1845}
219662023-09-22T23:34:13.569Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.352639285Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1845}
219672023-09-22T23:34:13.572Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.352672373Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1845}
219682023-09-22T23:34:13.575Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:13.352691563Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1845}
219692023-09-22T23:34:13.579ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
219702023-09-22T23:34:13.582Z 0 000-019 163 163 163 260 260 260 F F F
219712023-09-22T23:34:13.585ZError: Difference in extent metadata found!
219722023-09-22T23:34:13.588Z 1 020-039 163 163 163 260 260 260 F F F
219732023-09-22T23:34:13.591Z 2 040-059 161 161 161 258 258 258 F F F
219742023-09-22T23:34:13.594Z 3 060-079 164 164 164 263 263 263 F F F
219752023-09-22T23:34:13.596Z 4 080-099 166 163 166 266 260 266 F F F <---
219762023-09-22T23:34:13.600Z 5 100-119 163 163 163 260 260 260 F F F
219772023-09-22T23:34:13.603Z 6 120-139 166 166 166 268 265 268 F F F <---
219782023-09-22T23:34:13.605Z 7 140-159 166 164 166 267 263 267 F F F <---
219792023-09-22T23:34:13.609Z 8 160-179 157 157 157 250 250 250 F F F
219802023-09-22T23:34:13.612Z 9 180-199 164 164 164 263 263 263 F F F
219812023-09-22T23:34:13.616Z 10 200-219 166 153 166 267 242 267 F F F <---
219822023-09-22T23:34:13.619Z 11 220-239 164 164 164 262 262 262 F F F
219832023-09-22T23:34:13.622Z 12 240-259 166 166 166 265 265 265 F F F
219842023-09-22T23:34:13.625Z 13 260-279 164 164 164 263 263 263 F F F
219852023-09-22T23:34:13.628Z 14 280-299 164 164 164 263 263 263 F F F
219862023-09-22T23:34:13.631Z 15 300-319 165 165 165 264 264 264 F F F
219872023-09-22T23:34:13.634Z 16 320-339 158 158 158 252 252 252 F F F
219882023-09-22T23:34:13.637Z 17 340-359 163 163 163 260 260 260 F F F
219892023-09-22T23:34:13.641Z 18 360-379 166 166 166 268 265 268 F F F <---
219902023-09-22T23:34:13.644Z 19 380-399 166 160 166 268 256 268 T F T <---
219912023-09-22T23:34:13.649Z 20 400-419 166 163 166 266 260 266 T T T <---
219922023-09-22T23:34:13.652Z 21 420-439 166 165 166 266 264 266 F T F <---
219932023-09-22T23:34:13.655Z 22 440-459 165 165 165 264 264 264 F F F
219942023-09-22T23:34:13.659Z 23 460-479 161 161 161 258 258 258 F F F
219952023-09-22T23:34:13.662Z 24 480-499 164 164 164 263 263 263 T F T <---
219962023-09-22T23:34:13.666Z 25 500-519 154 154 154 247 247 247 T F T <---
219972023-09-22T23:34:13.669Z 26 520-539 165 165 165 264 264 264 T F F <---
219982023-09-22T23:34:13.672Z 27 540-559 166 160 166 268 257 268 F F F <---
219992023-09-22T23:34:13.676Z 28 560-579 166 160 166 268 257 268 F F F <---
220002023-09-22T23:34:13.679Z 29 580-599 166 166 166 265 265 265 F F F
220012023-09-22T23:34:13.682ZMax gen: 166, Max flush: 268
220022023-09-22T23:34:13.686ZA Difference in extent metadata is expected here
220032023-09-22T23:34:13.689ZOn loop 83
220042023-09-22T23:34:13.693Z
220052023-09-22T23:34:13.696ZVerifying data now
220062023-09-22T23:34:19.848ZFiles /tmp/test_repair_verify.data and /tmp/test_repair_verify.data.last differ
220072023-09-22T23:34:19.851Z138d137
220082023-09-22T23:34:19.854Z< 10,
220092023-09-22T23:34:19.857Z140d138
220102023-09-22T23:34:19.860Z< 13,
220112023-09-22T23:34:19.863Z141a140
220122023-09-22T23:34:19.866Z> 14,
220132023-09-22T23:34:19.869Z142a142,143
220142023-09-22T23:34:19.872Z> 14,
220152023-09-22T23:34:19.875Z> 14,
220162023-09-22T23:34:19.878Z144d144
220172023-09-22T23:34:19.881Z< 12,
220182023-09-22T23:34:19.884Zdiff found after verify
220192023-09-22T23:34:19.887ZLoop: 83 Downstairs dump after verify (and repair):
220202023-09-22T23:34:19.890Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.873344259Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1853}
220212023-09-22T23:34:19.893Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.873704151Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1853}
220222023-09-22T23:34:19.896Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.873754693Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1853}
220232023-09-22T23:34:19.900Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.892217103Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1853}
220242023-09-22T23:34:19.903Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.892266223Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1853}
220252023-09-22T23:34:19.906Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.892305307Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1853}
220262023-09-22T23:34:19.909Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.903161363Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1853}
220272023-09-22T23:34:19.912Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.903207582Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1853}
220282023-09-22T23:34:19.915Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.903246079Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1853}
220292023-09-22T23:34:19.918ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
220302023-09-22T23:34:19.921Z 0 000-019 163 163 163 260 260 260 F F F
220312023-09-22T23:34:19.924Z 1 020-039 163 163 163 260 260 260 F F F
220322023-09-22T23:34:19.927Z 2 040-059 161 161 161 258 258 258 F F F
220332023-09-22T23:34:19.930Z 3 060-079 164 164 164 263 263 263 F F F
220342023-09-22T23:34:19.933Z 4 080-099 166 166 166 266 266 266 F F F
220352023-09-22T23:34:19.937Z 5 100-119 163 163 163 260 260 260 F F F
220362023-09-22T23:34:19.940Z 6 120-139 166 166 166 268 268 268 F F F
220372023-09-22T23:34:19.943Z 7 140-159 166 166 166 267 267 267 F F F
220382023-09-22T23:34:19.946Z 8 160-179 157 157 157 250 250 250 F F F
220392023-09-22T23:34:19.949Z 9 180-199 164 164 164 263 263 263 F F F
220402023-09-22T23:34:19.952Z 10 200-219 166 166 166 267 267 267 F F F
220412023-09-22T23:34:19.955Z 11 220-239 164 164 164 262 262 262 F F F
220422023-09-22T23:34:19.958Z 12 240-259 166 166 166 265 265 265 F F F
220432023-09-22T23:34:19.961Z 13 260-279 164 164 164 263 263 263 F F F
220442023-09-22T23:34:19.964Z 14 280-299 164 164 164 263 263 263 F F F
220452023-09-22T23:34:19.967Z 15 300-319 165 165 165 264 264 264 F F F
220462023-09-22T23:34:19.970Z 16 320-339 158 158 158 252 252 252 F F F
220472023-09-22T23:34:19.973Z 17 340-359 163 163 163 260 260 260 F F F
220482023-09-22T23:34:19.976Z 18 360-379 166 166 166 268 268 268 F F F
220492023-09-22T23:34:19.979Z 19 380-399 167 167 167 269 269 269 F F F
220502023-09-22T23:34:19.982Z 20 400-419 167 167 167 269 269 269 F F F
220512023-09-22T23:34:19.985Z 21 420-439 166 166 166 266 266 266 F F F
220522023-09-22T23:34:19.988Z 22 440-459 165 165 165 264 264 264 F F F
220532023-09-22T23:34:19.990Z 23 460-479 161 161 161 258 258 258 F F F
220542023-09-22T23:34:19.993Z 24 480-499 167 167 167 269 269 269 F F F
220552023-09-22T23:34:19.996Z 25 500-519 167 167 167 269 269 269 F F F
220562023-09-22T23:34:19.999Z 26 520-539 167 167 167 269 269 269 F F F
220572023-09-22T23:34:20.003Z 27 540-559 166 166 166 268 268 268 F F F
220582023-09-22T23:34:20.006Z 28 560-579 166 166 166 268 268 268 F F F
220592023-09-22T23:34:20.009Z 29 580-599 166 166 166 265 265 265 F F F
220602023-09-22T23:34:20.012ZMax gen: 167, Max flush: 269
220612023-09-22T23:34:20.015Z
220622023-09-22T23:34:20.018ZBegin loop 84
220632023-09-22T23:34:20.021ZDownstairs to restart: 1
220642023-09-22T23:34:20.024Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.950427575Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220652023-09-22T23:34:20.026Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.950876323Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220662023-09-22T23:34:20.030Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.950930661Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220672023-09-22T23:34:20.033Z{"msg":"Crucible stats registered with UUID: 1fdcf7f2-faa2-4d92-afdc-bebe4362f199","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.950996757Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220682023-09-22T23:34:20.035ZCrucible runtime is spawned
220692023-09-22T23:34:20.038Z{"msg":"Crucible 1fdcf7f2-faa2-4d92-afdc-bebe4362f199 has session id: 32c460c8-a42a-4227-b5ae-a5915a8430da","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.951101646Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220702023-09-22T23:34:20.041Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.951207281Z","hostname":"The guest has requested activation with gen:168
220712023-09-22T23:34:20.044Zip-10-150-1-177.us-west-2.compute.internal","pid":1855,"task":"up_listen"}
220722023-09-22T23:34:20.048Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.951343438Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220732023-09-22T23:34:20.072Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.951412632Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220742023-09-22T23:34:20.075Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.951476803Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855,"looper":"2"}
220752023-09-22T23:34:20.078Z{"msg":"[2] 1fdcf7f2-faa2-4d92-afdc-bebe4362f199 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.951627294Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855,"looper":"2"}
220762023-09-22T23:34:20.081Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.951709259Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855,"looper":"1"}
220772023-09-22T23:34:20.084Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.951840195Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855,"looper":"0"}
220782023-09-22T23:34:20.087Z{"msg":"[0] 1fdcf7f2-faa2-4d92-afdc-bebe4362f199 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.952056849Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855,"looper":"0"}
220792023-09-22T23:34:20.090Z{"msg":"1fdcf7f2-faa2-4d92-afdc-bebe4362f199 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.95237845Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220802023-09-22T23:34:20.093Z{"msg":"Set desired generation to :168","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.952444523Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220812023-09-22T23:34:20.096Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.952593804Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220822023-09-22T23:34:20.099Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.952958091Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220832023-09-22T23:34:20.103Z{"msg":"[2] 1fdcf7f2-faa2-4d92-afdc-bebe4362f199 (e713f35c-c7b8-41bb-aca8-5292a50a9abc) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.953254215Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220842023-09-22T23:34:20.106Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.953285508Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220852023-09-22T23:34:20.108Z{"msg":"[2] client is_active_req TRUE, promote! session e713f35c-c7b8-41bb-aca8-5292a50a9abc","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.95332883Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220862023-09-22T23:34:20.112Z{"msg":"[0] 1fdcf7f2-faa2-4d92-afdc-bebe4362f199 (e713f35c-c7b8-41bb-aca8-5292a50a9abc) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.953604521Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220872023-09-22T23:34:20.114Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.953792935Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220882023-09-22T23:34:20.118Z{"msg":"[0] client is_active_req TRUE, promote! session e713f35c-c7b8-41bb-aca8-5292a50a9abc","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.953875882Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220892023-09-22T23:34:20.121Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.954060872Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220902023-09-22T23:34:20.124Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.954123965Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220912023-09-22T23:34:20.127Z{"msg":"1fdcf7f2-faa2-4d92-afdc-bebe4362f199 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.95425786Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220922023-09-22T23:34:20.130Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.954394987Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220932023-09-22T23:34:20.133Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.954458149Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220942023-09-22T23:34:20.137Z{"msg":"1fdcf7f2-faa2-4d92-afdc-bebe4362f199 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.954499779Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220952023-09-22T23:34:20.140Z{"msg":"[2] 1fdcf7f2-faa2-4d92-afdc-bebe4362f199 (e713f35c-c7b8-41bb-aca8-5292a50a9abc) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.956617703Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220962023-09-22T23:34:20.143Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.95667472Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220972023-09-22T23:34:20.146Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:34:19.956717664Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220982023-09-22T23:34:20.149Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.956756837Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
220992023-09-22T23:34:20.153Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.956879355Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221002023-09-22T23:34:20.155Z{"msg":"1fdcf7f2-faa2-4d92-afdc-bebe4362f199 WaitActive New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.956930829Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221012023-09-22T23:34:20.159Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.957114111Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221022023-09-22T23:34:20.162Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.957187526Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221032023-09-22T23:34:20.165Z{"msg":"[0] 1fdcf7f2-faa2-4d92-afdc-bebe4362f199 (e713f35c-c7b8-41bb-aca8-5292a50a9abc) WaitActive New WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.95726502Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221042023-09-22T23:34:20.168Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.957339686Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221052023-09-22T23:34:20.171Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:34:19.957412109Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221062023-09-22T23:34:20.174Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.957483283Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221072023-09-22T23:34:20.178Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.957552581Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221082023-09-22T23:34:20.181Z{"msg":"1fdcf7f2-faa2-4d92-afdc-bebe4362f199 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.957621156Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221092023-09-22T23:34:20.184Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.957657576Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221102023-09-22T23:34:20.187Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:19.957688503Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221112023-09-22T23:34:20.955Z{"msg":"[1] 1fdcf7f2-faa2-4d92-afdc-bebe4362f199 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.953177736Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855,"looper":"1"}
221122023-09-22T23:34:20.959Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.953267832Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221132023-09-22T23:34:20.962Z{"msg":"[1] 1fdcf7f2-faa2-4d92-afdc-bebe4362f199 (e713f35c-c7b8-41bb-aca8-5292a50a9abc) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.954476601Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221142023-09-22T23:34:20.965Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.954532044Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221152023-09-22T23:34:20.968Z{"msg":"[1] client is_active_req TRUE, promote! session e713f35c-c7b8-41bb-aca8-5292a50a9abc","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.954575802Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221162023-09-22T23:34:20.971Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.955145975Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221172023-09-22T23:34:20.975Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.955202687Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221182023-09-22T23:34:20.978Z{"msg":"1fdcf7f2-faa2-4d92-afdc-bebe4362f199 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.955245095Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221192023-09-22T23:34:20.981Z{"msg":"[1] 1fdcf7f2-faa2-4d92-afdc-bebe4362f199 (e713f35c-c7b8-41bb-aca8-5292a50a9abc) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.957742856Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221202023-09-22T23:34:20.984Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.957809604Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221212023-09-22T23:34:20.988Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:34:20.95785356Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221222023-09-22T23:34:20.991Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.957893531Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221232023-09-22T23:34:20.994Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.957914976Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221242023-09-22T23:34:20.997Z{"msg":"1fdcf7f2-faa2-4d92-afdc-bebe4362f199 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.957953641Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221252023-09-22T23:34:21.001Z{"msg":"[0]R flush_numbers[0..12]: [260, 260, 258, 263, 266, 260, 268, 267, 250, 263, 267, 262]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.958013685Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221262023-09-22T23:34:21.004Z{"msg":"The guest has finished waiting for activation with:168
221272023-09-22T23:34:21.007Z[0]R generation[0..12]: [163, 163, 161, 164, 166, 163, 166, 166, 157, 164, 166, 164]","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO
221282023-09-22T23:34:21.011Zlevel":30,"time":"2023-09-22T23:34:20.95809177Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221292023-09-22T23:34:21.013Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.958162102Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221302023-09-22T23:34:21.017Z{"msg":"[1]R flush_numbers[0..12]: [260, 260, 258, 263, 266, 260, 268, 267, 250, 263, 267, 262]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.958227157Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221312023-09-22T23:34:21.020Z{"msg":"[1]R generation[0..12]: [163, 163, 161, 164, 166, 163, 166, 166, 157, 164, 166, 164]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.95824967Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221322023-09-22T23:34:21.023Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.958304732Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221332023-09-22T23:34:21.026Z{"msg":"[2]R flush_numbers[0..12]: [260, 260, 258, 263, 266, 260, 268, 267, 250, 263, 267, 262]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.958376685Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221342023-09-22T23:34:21.030Z{"msg":"[2]R generation[0..12]: [163, 163, 161, 164, 166, 163, 166, 166, 157, 164, 166, 164]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.958427754Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221352023-09-22T23:34:21.033Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.958496879Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221362023-09-22T23:34:21.036Z{"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.958566685Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221372023-09-22T23:34:21.040Z{"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.958640463Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221382023-09-22T23:34:21.043Z{"msg":"Next flush: 270","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.958682087Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221392023-09-22T23:34:21.046Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.958721652Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221402023-09-22T23:34:21.049Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.958764599Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221412023-09-22T23:34:21.052Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.958809117Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221422023-09-22T23:34:21.055Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.958847766Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221432023-09-22T23:34:21.058Z{"msg":"1fdcf7f2-faa2-4d92-afdc-bebe4362f199 is now active with session: e713f35c-c7b8-41bb-aca8-5292a50a9abc","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.958900296Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221442023-09-22T23:34:21.062Z{"msg":"1fdcf7f2-faa2-4d92-afdc-bebe4362f199 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.958944816Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221452023-09-22T23:34:21.065Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.958984497Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221462023-09-22T23:34:21.068Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.959025336Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221472023-09-22T23:34:21.070Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.95906808Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221482023-09-22T23:34:21.093Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.959106922Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221492023-09-22T23:34:21.096Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.959146931Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221502023-09-22T23:34:21.099Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.959190402Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221512023-09-22T23:34:21.102Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.95923958Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221522023-09-22T23:34:21.105Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
221532023-09-22T23:34:21.108Z,"time":"2023-09-22T23:34:20.959294834Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221542023-09-22T23:34:21.111Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.959394095Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221552023-09-22T23:34:21.114Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.959440327Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221562023-09-22T23:34:21.117Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:20.959479007Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1855}
221572023-09-22T23:34:21.120ZLoading write count information from file "/tmp/test_repair_verify.data"
221582023-09-22T23:34:21.123ZRun Repair workload
221592023-09-22T23:34:21.126Z01/30 Write block 376 len 28672 data: 8 11 13 12 11 12 11
221602023-09-22T23:34:21.129Z02/30 Read block 323 len 4096
221612023-09-22T23:34:21.132Z03/30 Write block 338 len 8192 data: 14 13
221622023-09-22T23:34:21.135Z04/30 Read block 279 len 36864
221632023-09-22T23:34:21.138Z05/30 Write block 524 len 4096 data: 13
221642023-09-22T23:34:21.141Z06/30 Write block 558 len 28672 data: 9 11 14 12 12 12 14
221652023-09-22T23:34:21.144Z07/30 Write block 81 len 40960 data: 17 14 12 12 10 11 15 12 12 12
221662023-09-22T23:34:21.146Z08/30 Write block 276 len 32768 data: 21 18 20 21 21 19 19 15
221672023-09-22T23:34:21.149Z09/30 Read block 584 len 28672
221682023-09-22T23:34:21.152Z10/30 Write block 24 len 24576 data: 9 10 11 12 9 7
221692023-09-22T23:34:21.155Z11/30 Read block 510 len 36864
221702023-09-22T23:34:21.158Z12/30 Read block 299 len 24576
221712023-09-22T23:34:21.161Z13/30 Write block 292 len 4096 data: 21
221722023-09-22T23:34:21.165Z14/30 Write block 130 len 12288 data: 10 9 13
221732023-09-22T23:34:21.168Z15/30 Read block 146 len 28672
221742023-09-22T23:34:21.171Z16/30 Read block 122 len 24576
221752023-09-22T23:34:21.174Z17/30 Flush
221762023-09-22T23:34:21.177Z18/30 Write block 259 len 12288 data: 10 11 15
221772023-09-22T23:34:21.181Z19/30 Flush
221782023-09-22T23:34:21.184Z20/30 Write block 64 len 32768 data: 21 23 23 22 24 21 14 16
221792023-09-22T23:34:21.187Z21/30 Write block 167 len 8192 data: 9 8
221802023-09-22T23:34:21.190Z22/30 Flush
221812023-09-22T23:34:21.193Z23/30 Write block 543 len 4096 data: 11
221822023-09-22T23:34:21.196Z24/30 Read block 385 len 24576
221832023-09-22T23:34:21.199Z25/30 Read block 585 len 40960
221842023-09-22T23:34:21.203Z26/30 Write block 398 len 4096 data: 16
221852023-09-22T23:34:21.206Z27/30 Write block 538 len 20480 data: 17 17 12 9 10
221862023-09-22T23:34:21.209Z28/30 Write block 8 len 28672 data: 10 10 10 10 11 13 11
221872023-09-22T23:34:21.212Z29/30 Read block 42 len 20480
221882023-09-22T23:34:21.216Z30/30 Write block 417 len 36864 data: 12 16 17 15 14 11 11 11 7
221892023-09-22T23:34:21.218Z----------------------------------------------------------------
221902023-09-22T23:34:21.221Z Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:30
221912023-09-22T23:34:21.224ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
221922023-09-22T23:34:21.227Z 1 Acked 1000 Write 7 Done Done Done false
221932023-09-22T23:34:21.230Z 2 Acked 1001 Read 1 Done Sent Done false
221942023-09-22T23:34:21.233Z 3 Acked 1002 Write 2 Done Sent Done false
221952023-09-22T23:34:21.236Z 4 Acked 1003 Read 9 Done Sent Done false
221962023-09-22T23:34:21.239Z 5 Acked 1004 Write 1 Done Sent Done false
221972023-09-22T23:34:21.242Z 6 Acked 1005 Write 7 Done Sent Done false
221982023-09-22T23:34:21.245Z 7 Acked 1006 Write 10 Done Sent Done false
221992023-09-22T23:34:21.248Z 8 Acked 1007 Write 8 Done Sent Done false
222002023-09-22T23:34:21.251Z 9 Acked 1008 Read 7 Done Sent Done false
222012023-09-22T23:34:21.254Z 10 Acked 1009 Write 6 Done Sent Done false
222022023-09-22T23:34:21.257Z 11 Acked 1010 Read 9 Done Sent Done false
222032023-09-22T23:34:21.260Z 12 Acked 1011 Read 6 Done Sent Done false
222042023-09-22T23:34:21.263Z 13 Acked 1012 Write 1 Done Sent Done false
222052023-09-22T23:34:21.267Z 14 Acked 1013 Write 3 Done Sent Done false
222062023-09-22T23:34:21.269Z 15 Acked 1014 Read 7 Done Sent Done false
222072023-09-22T23:34:21.273Z 16 Acked 1015 Read 6 Done Sent Done false
222082023-09-22T23:34:21.276Z 17 Acked 1016 Flush 0 Done Sent Done false
222092023-09-22T23:34:21.279Z 18 Acked 1017 Write 3 Done Sent Done false
222102023-09-22T23:34:21.282Z 19 Acked 1018 Flush 0 Done Sent Done false
222112023-09-22T23:34:21.285Z 20 Acked 1019 Write 8 Done Sent Done false
222122023-09-22T23:34:21.288Z 21 Acked 1020 Write 2 Done Sent Done false
222132023-09-22T23:34:21.292Z 22 Acked 1021 Flush 0 Done Sent Done false
222142023-09-22T23:34:21.295Z 23 Acked 1022 Write 1 Done Sent Done false
222152023-09-22T23:34:21.298Z 24 Acked 1023 Read 6 Done Sent Done false
222162023-09-22T23:34:21.302Z 25 Acked 1024 Read 10 Done Sent Done false
222172023-09-22T23:34:21.305Z 26 Acked 1025 Write 1 Done Sent Done false
222182023-09-22T23:34:21.308Z 27 Acked 1026 Write 5 Done Sent Done false
222192023-09-22T23:34:21.311Z 28 Acked 1027 Write 7 Done Sent Done false
222202023-09-22T23:34:21.314Z 29 Acked 1028 Read 5 Done Sent Done false
222212023-09-22T23:34:21.317Z 30 Acked 1029 Write 9 Sent Sent Sent false
222222023-09-22T23:34:21.320Z STATES DS:0 DS:1 DS:2 TOTAL
222232023-09-22T23:34:21.323Z New 0 0 0 0
222242023-09-22T23:34:21.326Z Sent 1 29 1 31
222252023-09-22T23:34:21.330Z Done 29 1 29 59
222262023-09-22T23:34:21.332Z Skipped 0 0 0 0
222272023-09-22T23:34:21.335Z Error 0 0 0 0
222282023-09-22T23:34:21.339ZLast Flush: 1021 0 1021
222292023-09-22T23:34:21.342ZDownstairs last five completed:
222302023-09-22T23:34:21.345ZUpstairs last five completed: 30 29 28 27 26 25
222312023-09-22T23:34:21.348ZWrote out file "/tmp/test_repair_verify.data"
222322023-09-22T23:34:21.351Z
222332023-09-22T23:34:21.354ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
222342023-09-22T23:34:21.357Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:21.102548297Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1856}
222352023-09-22T23:34:21.360Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:21.102881922Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1856}
222362023-09-22T23:34:21.363Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:21.102905443Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1856}
222372023-09-22T23:34:21.366Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:21.130628789Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1856}
222382023-09-22T23:34:21.369Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:21.130663766Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1856}
222392023-09-22T23:34:21.372Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:21.130682972Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1856}
222402023-09-22T23:34:21.375Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:21.154402661Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1856}
222412023-09-22T23:34:21.378Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:21.154434962Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1856}
222422023-09-22T23:34:21.381ZError: Difference in extent metadata found!
222432023-09-22T23:34:21.385Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:21.1544536Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1856}
222442023-09-22T23:34:21.388ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
222452023-09-22T23:34:21.391Z 0 000-019 163 163 163 260 260 260 T F T <---
222462023-09-22T23:34:21.394Z 1 020-039 168 163 168 270 260 270 F F F <---
222472023-09-22T23:34:21.397Z 2 040-059 161 161 161 258 258 258 F F F
222482023-09-22T23:34:21.400Z 3 060-079 168 164 168 272 263 272 F F F <---
222492023-09-22T23:34:21.403Z 4 080-099 168 166 168 270 266 270 F F F <---
222502023-09-22T23:34:21.406Z 5 100-119 163 163 163 260 260 260 F F F
222512023-09-22T23:34:21.409Z 6 120-139 168 166 168 270 268 270 F F F <---
222522023-09-22T23:34:21.412Z 7 140-159 166 166 166 267 267 267 F F F
222532023-09-22T23:34:21.415Z 8 160-179 168 157 168 272 250 272 F F F <---
222542023-09-22T23:34:21.418Z 9 180-199 164 164 164 263 263 263 F F F
222552023-09-22T23:34:21.421Z 10 200-219 166 166 166 267 267 267 F F F
222562023-09-22T23:34:21.424Z 11 220-239 164 164 164 262 262 262 F F F
222572023-09-22T23:34:21.427Z 12 240-259 168 166 168 271 265 271 F F F <---
222582023-09-22T23:34:21.430Z 13 260-279 168 164 168 271 263 271 F F F <---
222592023-09-22T23:34:21.433Z 14 280-299 168 164 168 270 263 270 F F F <---
222602023-09-22T23:34:21.437Z 15 300-319 165 165 165 264 264 264 F F F
222612023-09-22T23:34:21.440Z 16 320-339 168 158 168 270 252 270 F F F <---
222622023-09-22T23:34:21.443Z 17 340-359 163 163 163 260 260 260 F F F
222632023-09-22T23:34:21.446Z 18 360-379 168 166 168 270 268 270 F T F <---
222642023-09-22T23:34:21.449Z 19 380-399 168 167 168 270 269 270 T T T <---
222652023-09-22T23:34:21.452Z 20 400-419 167 167 167 269 269 269 T F T <---
222662023-09-22T23:34:21.455Z 21 420-439 166 166 166 266 266 266 T F T <---
222672023-09-22T23:34:21.458Z 22 440-459 165 165 165 264 264 264 F F F
222682023-09-22T23:34:21.461Z 23 460-479 161 161 161 258 258 258 F F F
222692023-09-22T23:34:21.464Z 24 480-499 167 167 167 269 269 269 F F F
222702023-09-22T23:34:21.467Z 25 500-519 167 167 167 269 269 269 F F F
222712023-09-22T23:34:21.470Z 26 520-539 168 167 168 270 269 270 T F T <---
222722023-09-22T23:34:21.473Z 27 540-559 168 166 168 270 268 270 T F T <---
222732023-09-22T23:34:21.476Z 28 560-579 168 166 168 270 268 270 F F F <---
222742023-09-22T23:34:21.479Z 29 580-599 166 166 166 265 265 265 F F F
222752023-09-22T23:34:21.482ZMax gen: 168, Max flush: 272
222762023-09-22T23:34:21.485ZA Difference in extent metadata is expected here
222772023-09-22T23:34:21.488ZOn loop 84
222782023-09-22T23:34:21.491Z
222792023-09-22T23:34:21.494ZVerifying data now
222802023-09-22T23:34:27.911ZNo change after verify
222812023-09-22T23:34:27.915ZLoop: 84 Downstairs dump after verify (and repair):
222822023-09-22T23:34:27.930Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:27.927827966Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1861}
222832023-09-22T23:34:27.933Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:27.928213355Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1861}
222842023-09-22T23:34:27.936Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:27.928237937Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1861}
222852023-09-22T23:34:27.948Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:27.946769072Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1861}
222862023-09-22T23:34:27.951Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:27.946820609Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1861}
222872023-09-22T23:34:27.954Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:27.946841169Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1861}
222882023-09-22T23:34:27.959Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:27.957758954Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1861}
222892023-09-22T23:34:27.962Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:27.957811123Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1861}
222902023-09-22T23:34:27.965Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:27.957832081Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1861}
222912023-09-22T23:34:27.970ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
222922023-09-22T23:34:27.973Z 0 000-019 169 169 169 273 273 273 F F F
222932023-09-22T23:34:27.976Z 1 020-039 168 168 168 270 270 270 F F F
222942023-09-22T23:34:27.979Z 2 040-059 161 161 161 258 258 258 F F F
222952023-09-22T23:34:27.982Z 3 060-079 168 168 168 272 272 272 F F F
222962023-09-22T23:34:27.985Z 4 080-099 168 168 168 270 270 270 F F F
222972023-09-22T23:34:27.988Z 5 100-119 163 163 163 260 260 260 F F F
222982023-09-22T23:34:27.991Z 6 120-139 168 168 168 270 270 270 F F F
222992023-09-22T23:34:27.994Z 7 140-159 166 166 166 267 267 267 F F F
223002023-09-22T23:34:27.997Z 8 160-179 168 168 168 272 272 272 F F F
223012023-09-22T23:34:28.001Z 9 180-199 164 164 164 263 263 263 F F F
223022023-09-22T23:34:28.004Z 10 200-219 166 166 166 267 267 267 F F F
223032023-09-22T23:34:28.007Z 11 220-239 164 164 164 262 262 262 F F F
223042023-09-22T23:34:28.011Z 12 240-259 168 168 168 271 271 271 F F F
223052023-09-22T23:34:28.014Z 13 260-279 168 168 168 271 271 271 F F F
223062023-09-22T23:34:28.017Z 14 280-299 168 168 168 270 270 270 F F F
223072023-09-22T23:34:28.021Z 15 300-319 165 165 165 264 264 264 F F F
223082023-09-22T23:34:28.025Z 16 320-339 168 168 168 270 270 270 F F F
223092023-09-22T23:34:28.029Z 17 340-359 163 163 163 260 260 260 F F F
223102023-09-22T23:34:28.034Z 18 360-379 168 168 168 270 270 270 F F F
223112023-09-22T23:34:28.038Z 19 380-399 169 169 169 273 273 273 F F F
223122023-09-22T23:34:28.041Z 20 400-419 169 169 169 273 273 273 F F F
223132023-09-22T23:34:28.045Z 21 420-439 169 169 169 273 273 273 F F F
223142023-09-22T23:34:28.048Z 22 440-459 165 165 165 264 264 264 F F F
223152023-09-22T23:34:28.051Z 23 460-479 161 161 161 258 258 258 F F F
223162023-09-22T23:34:28.054Z 24 480-499 167 167 167 269 269 269 F F F
223172023-09-22T23:34:28.057Z 25 500-519 167 167 167 269 269 269 F F F
223182023-09-22T23:34:28.060Z 26 520-539 169 169 169 273 273 273 F F F
223192023-09-22T23:34:28.063Z 27 540-559 169 169 169 273 273 273 F F F
223202023-09-22T23:34:28.066Z 28 560-579 168 168 168 270 270 270 F F F
223212023-09-22T23:34:28.070Z 29 580-599 166 166 166 265 265 265 F F F
223222023-09-22T23:34:28.073ZMax gen: 169, Max flush: 273
223232023-09-22T23:34:28.076Z
223242023-09-22T23:34:28.079ZBegin loop 85
223252023-09-22T23:34:28.082ZDownstairs to restart: 1
223262023-09-22T23:34:28.085Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.004154996Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223272023-09-22T23:34:28.088Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.004524421Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223282023-09-22T23:34:28.091Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.004578328Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223292023-09-22T23:34:28.094Z{"msg":"Crucible stats registered with UUID: 4b628c89-a0d9-423b-98d1-d6bf5b8497f2","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.004666991Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223302023-09-22T23:34:28.101Z{"msg":"Crucible 4b628c89-a0d9-423b-98d1-d6bf5b8497f2 has session id: 426868af-c14d-44d0-b496-b2147258abc5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.004690622Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223312023-09-22T23:34:28.105ZCrucible runtime is spawned
223322023-09-22T23:34:28.108ZThe guest has requested activation with gen:170
223332023-09-22T23:34:28.111Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.005010157Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863,"task":"up_listen"}
223342023-09-22T23:34:28.114Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.005073608Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223352023-09-22T23:34:28.117Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.005120997Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223362023-09-22T23:34:28.121Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.005142878Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863,"looper":"0"}
223372023-09-22T23:34:28.124Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.005411806Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863,"looper":"2"}
223382023-09-22T23:34:28.127Z{"msg":"[0] 4b628c89-a0d9-423b-98d1-d6bf5b8497f2 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.005551739Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863,"looper":"0"}
223392023-09-22T23:34:28.130Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.005641016Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863,"looper":"1"}
223402023-09-22T23:34:28.133Z{"msg":"[2] 4b628c89-a0d9-423b-98d1-d6bf5b8497f2 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.00571763Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863,"looper":"2"}
223412023-09-22T23:34:28.136Z{"msg":"4b628c89-a0d9-423b-98d1-d6bf5b8497f2 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.005932701Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223422023-09-22T23:34:28.140Z{"msg":"Set desired generation to :170","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.005996834Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223432023-09-22T23:34:28.143Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.0062112Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223442023-09-22T23:34:28.147Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.006542255Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223452023-09-22T23:34:28.150Z{"msg":"[0] 4b628c89-a0d9-423b-98d1-d6bf5b8497f2 (4c24c6bd-945e-4b5d-a5c2-227f232f5948) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.00692332Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223462023-09-22T23:34:28.153Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.007002396Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223472023-09-22T23:34:28.157Z{"msg":"[0] client is_active_req TRUE, promote! session 4c24c6bd-945e-4b5d-a5c2-227f232f5948","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.007058003Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223482023-09-22T23:34:28.159Z{"msg":"[2] 4b628c89-a0d9-423b-98d1-d6bf5b8497f2 (4c24c6bd-945e-4b5d-a5c2-227f232f5948) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.007162638Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223492023-09-22T23:34:28.162Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.007240199Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223502023-09-22T23:34:28.167Z{"msg":"[2] client is_active_req TRUE, promote! session 4c24c6bd-945e-4b5d-a5c2-227f232f5948","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.007316491Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223512023-09-22T23:34:28.172Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.007658841Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223522023-09-22T23:34:28.175Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.007721094Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223532023-09-22T23:34:28.178Z{"msg":"4b628c89-a0d9-423b-98d1-d6bf5b8497f2 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.007847159Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223542023-09-22T23:34:28.181Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.008005521Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223552023-09-22T23:34:28.185Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.00804121Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223562023-09-22T23:34:28.188Z{"msg":"4b628c89-a0d9-423b-98d1-d6bf5b8497f2 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.008106853Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223572023-09-22T23:34:28.191Z{"msg":"[0] 4b628c89-a0d9-423b-98d1-d6bf5b8497f2 (4c24c6bd-945e-4b5d-a5c2-227f232f5948) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.010085017Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223582023-09-22T23:34:28.193Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.010146931Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223592023-09-22T23:34:28.196Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:34:28.010221856Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223602023-09-22T23:34:28.199Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.010327872Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223612023-09-22T23:34:28.202Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.010371867Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223622023-09-22T23:34:28.205Z{"msg":"4b628c89-a0d9-423b-98d1-d6bf5b8497f2 WaitQuorum New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.01041189Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223632023-09-22T23:34:28.209Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.010450515Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223642023-09-22T23:34:28.212Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.010488843Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223652023-09-22T23:34:28.215Z{"msg":"[2] 4b628c89-a0d9-423b-98d1-d6bf5b8497f2 (4c24c6bd-945e-4b5d-a5c2-227f232f5948) WaitQuorum New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.010529816Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223662023-09-22T23:34:28.218Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.010568668Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223672023-09-22T23:34:28.220Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:34:28.010607348Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223682023-09-22T23:34:28.223Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.010645413Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223692023-09-22T23:34:28.226Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.010683845Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223702023-09-22T23:34:28.229Z{"msg":"4b628c89-a0d9-423b-98d1-d6bf5b8497f2 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.010722401Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223712023-09-22T23:34:28.252Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.010742253Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223722023-09-22T23:34:28.256Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:28.010759737Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223732023-09-22T23:34:29.009Z{"msg":"[1] 4b628c89-a0d9-423b-98d1-d6bf5b8497f2 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.006451463Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863,"looper":"1"}
223742023-09-22T23:34:29.012Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.00654331Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223752023-09-22T23:34:29.015Z{"msg":"[1] 4b628c89-a0d9-423b-98d1-d6bf5b8497f2 (4c24c6bd-945e-4b5d-a5c2-227f232f5948) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.007717853Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223762023-09-22T23:34:29.019Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.007780172Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223772023-09-22T23:34:29.022Z{"msg":"[1] client is_active_req TRUE, promote! session 4c24c6bd-945e-4b5d-a5c2-227f232f5948","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.007825511Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223782023-09-22T23:34:29.030Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.008378108Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223792023-09-22T23:34:29.034Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.008435467Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223802023-09-22T23:34:29.037Z{"msg":"4b628c89-a0d9-423b-98d1-d6bf5b8497f2 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.00847997Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223812023-09-22T23:34:29.040Z{"msg":"[1] 4b628c89-a0d9-423b-98d1-d6bf5b8497f2 (4c24c6bd-945e-4b5d-a5c2-227f232f5948) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.010881023Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223822023-09-22T23:34:29.043Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.010961441Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223832023-09-22T23:34:29.046Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:34:29.01101058Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223842023-09-22T23:34:29.051Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.011050607Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223852023-09-22T23:34:29.054Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.01108921Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223862023-09-22T23:34:29.058Z{"msg":"4b628c89-a0d9-423b-98d1-d6bf5b8497f2 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.01112956Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223872023-09-22T23:34:29.061Z{"msg":"[0]R flush_numbers[0..12]: [273, 270, 258, 272, 270, 260, 270, 267, 272, 263, 267, 262]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.011169432Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223882023-09-22T23:34:29.064Z{"msg":"[0]R generation[0..12]: [169, 168, 161, 168, 168, 163, 168, 166, 168, 164, 166, 164]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.011209951Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223892023-09-22T23:34:29.067Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.011263141Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223902023-09-22T23:34:29.070Z{"msg":"[1]R flush_numbers[0..12]: [273, 270, 258, 272, 270, 260, 270, 267, 272, 263, 267, 262]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.011303583Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863The guest has finished waiting for activation with:170
223912023-09-22T23:34:29.073ZWait for a query_work_queue command to finish before sending IO
223922023-09-22T23:34:29.076Z}
223932023-09-22T23:34:29.079Z{"msg":"[1]R generation[0..12]: [169, 168, 161, 168, 168, 163, 168, 166, 168, 164, 166, 164]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.011469022Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223942023-09-22T23:34:29.082Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.01157775Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223952023-09-22T23:34:29.085Z{"msg":"[2]R flush_numbers[0..12]: [273, 270, 258, 272, 270, 260, 270, 267, 272, 263, 267, 262]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.011636573Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223962023-09-22T23:34:29.088Z{"msg":"[2]R generation[0..12]: [169, 168, 161, 168, 168, 163, 168, 166, 168, 164, 166, 164]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.011679577Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223972023-09-22T23:34:29.091Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.011703455Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223982023-09-22T23:34:29.095Z{"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.01174802Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
223992023-09-22T23:34:29.098Z{"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.011787593Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
224002023-09-22T23:34:29.101Z{"msg":"Next flush: 274","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.011828165Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
224012023-09-22T23:34:29.104Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.011867635Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
224022023-09-22T23:34:29.107Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.011887676Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
224032023-09-22T23:34:29.110Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.011931733Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
224042023-09-22T23:34:29.113Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.011987953Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
224052023-09-22T23:34:29.116Z{"msg":"4b628c89-a0d9-423b-98d1-d6bf5b8497f2 is now active with session: 4c24c6bd-945e-4b5d-a5c2-227f232f5948","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.012043045Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
224062023-09-22T23:34:29.120Z{"msg":"4b628c89-a0d9-423b-98d1-d6bf5b8497f2 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.012091963Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
224072023-09-22T23:34:29.123Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.012113878Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
224082023-09-22T23:34:29.127Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.012149967Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
224092023-09-22T23:34:29.130Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.012170004Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
224102023-09-22T23:34:29.133Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.012206295Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
224112023-09-22T23:34:29.136Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.012225554Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
224122023-09-22T23:34:29.139Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.01227396Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
224132023-09-22T23:34:29.142Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.012312148Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
224142023-09-22T23:34:29.145Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.012332837Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
224152023-09-22T23:34:29.148Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.012368689Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
224162023-09-22T23:34:29.151Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.012388325Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
224172023-09-22T23:34:29.154Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
224182023-09-22T23:34:29.157Z,"time":"2023-09-22T23:34:29.012428032Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1863}
224192023-09-22T23:34:29.160ZLoading write count information from file "/tmp/test_repair_verify.data"
224202023-09-22T23:34:29.163ZRun Repair workload
224212023-09-22T23:34:29.166Z01/30 Write block 250 len 16384 data: 9 8 12 11
224222023-09-22T23:34:29.169Z02/30 Flush
224232023-09-22T23:34:29.172Z03/30 Write block 497 len 40960 data: 19 13 13 13 12 13 12 12 12 13
224242023-09-22T23:34:29.175Z04/30 Read block 299 len 40960
224252023-09-22T23:34:29.178Z05/30 Write block 490 len 8192 data: 22 23
224262023-09-22T23:34:29.181Z06/30 Read block 503 len 28672
224272023-09-22T23:34:29.186Z07/30 Write block 219 len 12288 data: 9 11 10
224282023-09-22T23:34:29.188Z08/30 Read block 94 len 24576
224292023-09-22T23:34:29.191Z09/30 Write block 479 len 24576 data: 12 12 11 14 13 18
224302023-09-22T23:34:29.194Z10/30 Read block 484 len 36864
224312023-09-22T23:34:29.197Z11/30 Write block 56 len 12288 data: 6 9 12
224322023-09-22T23:34:29.201Z12/30 Read block 253 len 40960
224332023-09-22T23:34:29.204Z13/30 Read block 465 len 8192
224342023-09-22T23:34:29.207Z14/30 Read block 143 len 40960
224352023-09-22T23:34:29.209Z15/30 Read block 31 len 32768
224362023-09-22T23:34:29.212Z16/30 Read block 359 len 12288
224372023-09-22T23:34:29.216Z17/30 Read block 115 len 40960
224382023-09-22T23:34:29.219Z18/30 Flush
224392023-09-22T23:34:29.222Z19/30 Write block 62 len 16384 data: 19 22 22 24
224402023-09-22T23:34:29.225Z20/30 Read block 189 len 24576
224412023-09-22T23:34:29.228Z21/30 Read block 496 len 16384
224422023-09-22T23:34:29.231Z22/30 Read block 204 len 4096
224432023-09-22T23:34:29.234Z23/30 Read block 376 len 16384
224442023-09-22T23:34:29.237Z24/30 Write block 75 len 28672 data: 15 14 15 15 19 16 18
224452023-09-22T23:34:29.240Z25/30 Read block 413 len 24576
224462023-09-22T23:34:29.243Z26/30 Write block 423 len 4096 data: 12
224472023-09-22T23:34:29.246Z27/30 Write block 467 len 36864 data: 15 16 15 13 11 12 11 11 10
224482023-09-22T23:34:29.249Z28/30 Read block 432 len 40960
224492023-09-22T23:34:29.274Z29/30 Write block 358 len 16384 data: 15 17 18 14
224502023-09-22T23:34:29.277Z30/30 Write block 515 len 12288 data: 18 16 17
224512023-09-22T23:34:29.280Z----------------------------------------------------------------
224522023-09-22T23:34:29.283Z Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30
224532023-09-22T23:34:29.286ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
224542023-09-22T23:34:29.290Z 1 Acked 1000 Write 4 Done Sent Done false
224552023-09-22T23:34:29.293Z 2 Acked 1001 Flush 0 Done Sent Done false
224562023-09-22T23:34:29.296Z 3 Acked 1002 Write 10 Done Sent Done false
224572023-09-22T23:34:29.299Z 4 Acked 1003 Read 10 Done Sent Done false
224582023-09-22T23:34:29.302Z 5 Acked 1004 Write 2 Done Sent Done false
224592023-09-22T23:34:29.305Z 6 Acked 1005 Read 7 Done Sent Done false
224602023-09-22T23:34:29.308Z 7 Acked 1006 Write 3 Done Sent Done false
224612023-09-22T23:34:29.311Z 8 Acked 1007 Read 6 Done Sent Done false
224622023-09-22T23:34:29.314Z 9 Acked 1008 Write 6 Done Sent Done false
224632023-09-22T23:34:29.317Z 10 Acked 1009 Read 9 Done Sent Done false
224642023-09-22T23:34:29.320Z 11 Acked 1010 Write 3 Done Sent Done false
224652023-09-22T23:34:29.323Z 12 Acked 1011 Read 10 Done Sent Done false
224662023-09-22T23:34:29.327Z 13 Acked 1012 Read 2 Done Sent Done false
224672023-09-22T23:34:29.330Z 14 Acked 1013 Read 10 Done Sent Done false
224682023-09-22T23:34:29.333Z 15 Acked 1014 Read 8 Done Sent Done false
224692023-09-22T23:34:29.336Z 16 Acked 1015 Read 3 Done Sent Done false
224702023-09-22T23:34:29.339Z 17 Acked 1016 Read 10 Done Sent Done false
224712023-09-22T23:34:29.342Z 18 Acked 1017 Flush 0 Done Sent Done false
224722023-09-22T23:34:29.345Z 19 Acked 1018 Write 4 Done Sent Done false
224732023-09-22T23:34:29.348Z 20 Acked 1019 Read 6 Done Sent Done false
224742023-09-22T23:34:29.351Z 21 Acked 1020 Read 4 Done Sent Done false
224752023-09-22T23:34:29.354Z 22 Acked 1021 Read 1 Done Sent Done false
224762023-09-22T23:34:29.357Z 23 Acked 1022 Read 4 Done Sent Done false
224772023-09-22T23:34:29.360Z 24 Acked 1023 Write 7 Done Sent Done false
224782023-09-22T23:34:29.363Z 25 Acked 1024 Read 6 Done Sent Done false
224792023-09-22T23:34:29.366Z 26 Acked 1025 Write 1 Done Sent Done false
224802023-09-22T23:34:29.369Z 27 Acked 1026 Write 9 Done Sent Done false
224812023-09-22T23:34:29.372Z 28 Acked 1027 Read 10 Done Sent Done false
224822023-09-22T23:34:29.376Z 29 Acked 1028 Write 4 Sent Sent Sent false
224832023-09-22T23:34:29.379Z 30 Acked 1029 Write 3 Sent Sent Sent false
224842023-09-22T23:34:29.382Z STATES DS:0 DS:1 DS:2 TOTAL
224852023-09-22T23:34:29.385Z New 0 0 0 0
224862023-09-22T23:34:29.388Z Sent 2 30 2 34
224872023-09-22T23:34:29.391Z Done 28 0 28 56
224882023-09-22T23:34:29.395Z Skipped 0 0 0 0
224892023-09-22T23:34:29.398Z Error 0 0 0 0
224902023-09-22T23:34:29.401ZLast Flush: 1017 0 1017
224912023-09-22T23:34:29.404ZDownstairs last five completed:
224922023-09-22T23:34:29.407ZUpstairs last five completed: 30 29 28 27 26 25
224932023-09-22T23:34:29.411ZWrote out file "/tmp/test_repair_verify.data"
224942023-09-22T23:34:29.414Z
224952023-09-22T23:34:29.417ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
224962023-09-22T23:34:29.421Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.14810534Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1864}
224972023-09-22T23:34:29.424Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.148432791Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1864}
224982023-09-22T23:34:29.427Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.148456263Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1864}
224992023-09-22T23:34:29.430Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.177568943Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1864}
225002023-09-22T23:34:29.433Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.177601626Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1864}
225012023-09-22T23:34:29.437Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.177621007Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1864}
225022023-09-22T23:34:29.441Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.192929612Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1864}
225032023-09-22T23:34:29.444ZError: Difference in extent metadata found!
225042023-09-22T23:34:29.447Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.192960455Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1864}
225052023-09-22T23:34:29.450Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:29.192980026Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1864}
225062023-09-22T23:34:29.454ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
225072023-09-22T23:34:29.457Z 0 000-019 169 169 169 273 273 273 F F F
225082023-09-22T23:34:29.460Z 1 020-039 168 168 168 270 270 270 F F F
225092023-09-22T23:34:29.463Z 2 040-059 170 161 170 275 258 275 F F F <---
225102023-09-22T23:34:29.465Z 3 060-079 168 168 168 272 272 272 T F T <---
225112023-09-22T23:34:29.468Z 4 080-099 168 168 168 270 270 270 T F T <---
225122023-09-22T23:34:29.471Z 5 100-119 163 163 163 260 260 260 F F F
225132023-09-22T23:34:29.474Z 6 120-139 168 168 168 270 270 270 F F F
225142023-09-22T23:34:29.477Z 7 140-159 166 166 166 267 267 267 F F F
225152023-09-22T23:34:29.480Z 8 160-179 168 168 168 272 272 272 F F F
225162023-09-22T23:34:29.483Z 9 180-199 164 164 164 263 263 263 F F F
225172023-09-22T23:34:29.486Z 10 200-219 170 166 170 275 267 275 F F F <---
225182023-09-22T23:34:29.489Z 11 220-239 170 164 170 275 262 275 F F F <---
225192023-09-22T23:34:29.493Z 12 240-259 170 168 170 274 271 274 F F F <---
225202023-09-22T23:34:29.495Z 13 260-279 168 168 168 271 271 271 F F F
225212023-09-22T23:34:29.498Z 14 280-299 168 168 168 270 270 270 F F F
225222023-09-22T23:34:29.501Z 15 300-319 165 165 165 264 264 264 F F F
225232023-09-22T23:34:29.504Z 16 320-339 168 168 168 270 270 270 F F F
225242023-09-22T23:34:29.507Z 17 340-359 163 163 163 260 260 260 T F T <---
225252023-09-22T23:34:29.510Z 18 360-379 168 168 168 270 270 270 T F T <---
225262023-09-22T23:34:29.513Z 19 380-399 169 169 169 273 273 273 F F F
225272023-09-22T23:34:29.516Z 20 400-419 169 169 169 273 273 273 F F F
225282023-09-22T23:34:29.519Z 21 420-439 169 169 169 273 273 273 T F T <---
225292023-09-22T23:34:29.522Z 22 440-459 165 165 165 264 264 264 F F F
225302023-09-22T23:34:29.525Z 23 460-479 170 161 170 275 258 275 T F T <---
225312023-09-22T23:34:29.528Z 24 480-499 170 167 170 275 269 275 F F F <---
225322023-09-22T23:34:29.531Z 25 500-519 170 167 170 275 269 275 T F T <---
225332023-09-22T23:34:29.534Z 26 520-539 169 169 169 273 273 273 F F F
225342023-09-22T23:34:29.537Z 27 540-559 169 169 169 273 273 273 F F F
225352023-09-22T23:34:29.540Z 28 560-579 168 168 168 270 270 270 F F F
225362023-09-22T23:34:29.543Z 29 580-599 166 166 166 265 265 265 F F F
225372023-09-22T23:34:29.546ZMax gen: 170, Max flush: 275
225382023-09-22T23:34:29.549ZA Difference in extent metadata is expected here
225392023-09-22T23:34:29.552ZOn loop 85
225402023-09-22T23:34:29.556Z
225412023-09-22T23:34:29.559ZVerifying data now
225422023-09-22T23:34:35.537ZNo change after verify
225432023-09-22T23:34:35.540ZLoop: 85 Downstairs dump after verify (and repair):
225442023-09-22T23:34:35.556Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.553737891Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1869}
225452023-09-22T23:34:35.560Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.554137869Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1869}
225462023-09-22T23:34:35.563Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.554162731Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1869}
225472023-09-22T23:34:35.574Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.573102353Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1869}
225482023-09-22T23:34:35.578Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.573155352Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1869}
225492023-09-22T23:34:35.581Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.57317613Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1869}
225502023-09-22T23:34:35.586Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.584323083Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1869}
225512023-09-22T23:34:35.589Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.584371811Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1869}
225522023-09-22T23:34:35.592Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.584392299Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1869}
225532023-09-22T23:34:35.597ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
225542023-09-22T23:34:35.600Z 0 000-019 169 169 169 273 273 273 F F F
225552023-09-22T23:34:35.604Z 1 020-039 168 168 168 270 270 270 F F F
225562023-09-22T23:34:35.607Z 2 040-059 170 170 170 275 275 275 F F F
225572023-09-22T23:34:35.610Z 3 060-079 171 171 171 276 276 276 F F F
225582023-09-22T23:34:35.613Z 4 080-099 171 171 171 276 276 276 F F F
225592023-09-22T23:34:35.616Z 5 100-119 163 163 163 260 260 260 F F F
225602023-09-22T23:34:35.619Z 6 120-139 168 168 168 270 270 270 F F F
225612023-09-22T23:34:35.623Z 7 140-159 166 166 166 267 267 267 F F F
225622023-09-22T23:34:35.626Z 8 160-179 168 168 168 272 272 272 F F F
225632023-09-22T23:34:35.629Z 9 180-199 164 164 164 263 263 263 F F F
225642023-09-22T23:34:35.632Z 10 200-219 170 170 170 275 275 275 F F F
225652023-09-22T23:34:35.635Z 11 220-239 170 170 170 275 275 275 F F F
225662023-09-22T23:34:35.638Z 12 240-259 170 170 170 274 274 274 F F F
225672023-09-22T23:34:35.643Z 13 260-279 168 168 168 271 271 271 F F F
225682023-09-22T23:34:35.646Z 14 280-299 168 168 168 270 270 270 F F F
225692023-09-22T23:34:35.649Z 15 300-319 165 165 165 264 264 264 F F F
225702023-09-22T23:34:35.653Z 16 320-339 168 168 168 270 270 270 F F F
225712023-09-22T23:34:35.656Z 17 340-359 171 171 171 276 276 276 F F F
225722023-09-22T23:34:35.659Z 18 360-379 171 171 171 276 276 276 F F F
225732023-09-22T23:34:35.663Z 19 380-399 169 169 169 273 273 273 F F F
225742023-09-22T23:34:35.666Z 20 400-419 169 169 169 273 273 273 F F F
225752023-09-22T23:34:35.669Z 21 420-439 171 171 171 276 276 276 F F F
225762023-09-22T23:34:35.672Z 22 440-459 165 165 165 264 264 264 F F F
225772023-09-22T23:34:35.676Z 23 460-479 171 171 171 276 276 276 F F F
225782023-09-22T23:34:35.680Z 24 480-499 170 170 170 275 275 275 F F F
225792023-09-22T23:34:35.685Z 25 500-519 171 171 171 276 276 276 F F F
225802023-09-22T23:34:35.688Z 26 520-539 169 169 169 273 273 273 F F F
225812023-09-22T23:34:35.692Z 27 540-559 169 169 169 273 273 273 F F F
225822023-09-22T23:34:35.695Z 28 560-579 168 168 168 270 270 270 F F F
225832023-09-22T23:34:35.698Z 29 580-599 166 166 166 265 265 265 F F F
225842023-09-22T23:34:35.701ZMax gen: 171, Max flush: 276
225852023-09-22T23:34:35.704Z
225862023-09-22T23:34:35.707ZBegin loop 86
225872023-09-22T23:34:35.710ZDownstairs to restart: 2
225882023-09-22T23:34:35.713Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.633283199Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
225892023-09-22T23:34:35.716Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.633713621Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
225902023-09-22T23:34:35.719Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.633766965Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
225912023-09-22T23:34:35.722Z{"msg":"Crucible stats registered with UUID: bc242db9-9ef7-4f79-a537-24100cd59fc9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.633835075Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
225922023-09-22T23:34:35.725Z{"msg":"Crucible bc242db9-9ef7-4f79-a537-24100cd59fc9 has session id: be14043f-35b4-4ec4-96fa-50ebe00a852d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.633909034Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
225932023-09-22T23:34:35.728ZCrucible runtime is spawned
225942023-09-22T23:34:35.731ZThe guest has requested activation with gen:172
225952023-09-22T23:34:35.735Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.634186059Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871,"task":"up_listen"}
225962023-09-22T23:34:35.738Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.634267354Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
225972023-09-22T23:34:35.741Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.634314531Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
225982023-09-22T23:34:35.746Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.634337024Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871,"looper":"1"}
225992023-09-22T23:34:35.750Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.634517394Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871,"looper":"2"}
226002023-09-22T23:34:35.753Z{"msg":"[1] bc242db9-9ef7-4f79-a537-24100cd59fc9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.634617052Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871,"looper":"1"}
226012023-09-22T23:34:35.756Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.634665372Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871,"looper":"0"}
226022023-09-22T23:34:35.760Z{"msg":"[0] bc242db9-9ef7-4f79-a537-24100cd59fc9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.634815535Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871,"looper":"0"}
226032023-09-22T23:34:35.763Z{"msg":"bc242db9-9ef7-4f79-a537-24100cd59fc9 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.63521441Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226042023-09-22T23:34:35.768Z{"msg":"Set desired generation to :172","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.635275538Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226052023-09-22T23:34:35.771Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.635337376Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226062023-09-22T23:34:35.774Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.635659221Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226072023-09-22T23:34:35.777Z{"msg":"[1] bc242db9-9ef7-4f79-a537-24100cd59fc9 (16268435-fd17-401c-a014-94b11b26449e) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.636090977Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226082023-09-22T23:34:35.780Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.636155257Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226092023-09-22T23:34:35.784Z{"msg":"[1] client is_active_req TRUE, promote! session 16268435-fd17-401c-a014-94b11b26449e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.63621481Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226102023-09-22T23:34:35.789Z{"msg":"[0] bc242db9-9ef7-4f79-a537-24100cd59fc9 (16268435-fd17-401c-a014-94b11b26449e) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.636319316Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226112023-09-22T23:34:35.792Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.636382827Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226122023-09-22T23:34:35.795Z{"msg":"[0] client is_active_req TRUE, promote! session 16268435-fd17-401c-a014-94b11b26449e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.636428869Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226132023-09-22T23:34:35.798Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.636811788Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226142023-09-22T23:34:35.801Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.636872068Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226152023-09-22T23:34:35.805Z{"msg":"bc242db9-9ef7-4f79-a537-24100cd59fc9 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.637098714Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226162023-09-22T23:34:35.807Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.637240987Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226172023-09-22T23:34:35.810Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.637344462Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226182023-09-22T23:34:35.814Z{"msg":"bc242db9-9ef7-4f79-a537-24100cd59fc9 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.637431141Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226192023-09-22T23:34:35.817Z{"msg":"[1] bc242db9-9ef7-4f79-a537-24100cd59fc9 (16268435-fd17-401c-a014-94b11b26449e) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.639086005Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226202023-09-22T23:34:35.820Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.639189208Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226212023-09-22T23:34:35.825Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:34:35.639246528Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226222023-09-22T23:34:35.829Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.639296898Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226232023-09-22T23:34:35.832Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.639337129Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226242023-09-22T23:34:35.835Z{"msg":"bc242db9-9ef7-4f79-a537-24100cd59fc9 WaitActive WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.639359904Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226252023-09-22T23:34:35.838Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.639397121Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226262023-09-22T23:34:35.841Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.639449362Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226272023-09-22T23:34:35.844Z{"msg":"[0] bc242db9-9ef7-4f79-a537-24100cd59fc9 (16268435-fd17-401c-a014-94b11b26449e) WaitActive WaitQuorum New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.639507563Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226282023-09-22T23:34:35.847Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.639550261Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226292023-09-22T23:34:35.850Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:34:35.639589052Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226302023-09-22T23:34:35.854Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.639627265Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226312023-09-22T23:34:35.857Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.639665527Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226322023-09-22T23:34:35.859Z{"msg":"bc242db9-9ef7-4f79-a537-24100cd59fc9 WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.639704089Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226332023-09-22T23:34:35.863Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.63972404Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226342023-09-22T23:34:35.866Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:35.63974175Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226352023-09-22T23:34:36.638Z{"msg":"[2] bc242db9-9ef7-4f79-a537-24100cd59fc9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.635625813Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871,"looper":"2"}
226362023-09-22T23:34:36.641Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.635710526Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226372023-09-22T23:34:36.644Z{"msg":"[2] bc242db9-9ef7-4f79-a537-24100cd59fc9 (16268435-fd17-401c-a014-94b11b26449e) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.63684777Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226382023-09-22T23:34:36.647Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.636902719Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226392023-09-22T23:34:36.650Z{"msg":"[2] client is_active_req TRUE, promote! session 16268435-fd17-401c-a014-94b11b26449e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.636930176Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226402023-09-22T23:34:36.653Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.63755124Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226412023-09-22T23:34:36.657Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.637609988Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226422023-09-22T23:34:36.661Z{"msg":"bc242db9-9ef7-4f79-a537-24100cd59fc9 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.637652195Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226432023-09-22T23:34:36.664Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.639971631Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226442023-09-22T23:34:36.667Z{"msg":"[2] bc242db9-9ef7-4f79-a537-24100cd59fc9 (16268435-fd17-401c-a014-94b11b26449e) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.640125297Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226452023-09-22T23:34:36.670Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.640220054Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226462023-09-22T23:34:36.673Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:34:36.640274921Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226472023-09-22T23:34:36.676Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.640316312Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226482023-09-22T23:34:36.679Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.640355914Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226492023-09-22T23:34:36.683Z{"msg":"bc242db9-9ef7-4f79-a537-24100cd59fc9 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.640396348Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226502023-09-22T23:34:36.685Z{"msg":"[0]R flush_numbers[0..12]: [273, 270, 275, 276, 276, 260, 270, 267, 272, 263, 275, 275]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.640441634Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226512023-09-22T23:34:36.688Z{"msg":"[0]R generation[0..12]: [169, 168, 170, 171, 171, 163, 168, 166, 168, 164, 170, 170]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.640482291Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226522023-09-22T23:34:36.691Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:172
226532023-09-22T23:34:36.695ZWait for a query_work_queue command to finish before sending IO
226542023-09-22T23:34:36.698Z,"time":"2023-09-22T23:34:36.640545875Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226552023-09-22T23:34:36.701Z{"msg":"[1]R flush_numbers[0..12]: [273, 270, 275, 276, 276, 260, 270, 267, 272, 263, 275, 275]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.640662333Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226562023-09-22T23:34:36.704Z{"msg":"[1]R generation[0..12]: [169, 168, 170, 171, 171, 163, 168, 166, 168, 164, 170, 170]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.640736573Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226572023-09-22T23:34:36.707Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.640801762Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226582023-09-22T23:34:36.710Z{"msg":"[2]R flush_numbers[0..12]: [273, 270, 275, 276, 276, 260, 270, 267, 272, 263, 275, 275]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.640850524Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226592023-09-22T23:34:36.713Z{"msg":"[2]R generation[0..12]: [169, 168, 170, 171, 171, 163, 168, 166, 168, 164, 170, 170]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.640874086Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226602023-09-22T23:34:36.716Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.640913162Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226612023-09-22T23:34:36.720Z{"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.640952367Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226622023-09-22T23:34:36.723Z{"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.640992478Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226632023-09-22T23:34:36.726Z{"msg":"Next flush: 277","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.64103755Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226642023-09-22T23:34:36.730Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.641058964Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226652023-09-22T23:34:36.733Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.641095243Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226662023-09-22T23:34:36.736Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.641136369Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226672023-09-22T23:34:36.739Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.641155959Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226682023-09-22T23:34:36.742Z{"msg":"bc242db9-9ef7-4f79-a537-24100cd59fc9 is now active with session: 16268435-fd17-401c-a014-94b11b26449e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.641175211Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226692023-09-22T23:34:36.746Z{"msg":"bc242db9-9ef7-4f79-a537-24100cd59fc9 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.64122113Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226702023-09-22T23:34:36.749Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.641260452Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226712023-09-22T23:34:36.752Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.641279875Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226722023-09-22T23:34:36.755Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.641316676Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226732023-09-22T23:34:36.759Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.641336162Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226742023-09-22T23:34:36.762Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.641380811Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226752023-09-22T23:34:36.765Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.641400194Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226762023-09-22T23:34:36.768Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.641444001Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226772023-09-22T23:34:36.771Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.641482745Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226782023-09-22T23:34:36.776Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.641502104Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226792023-09-22T23:34:36.779Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.64151929Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226802023-09-22T23:34:36.783Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.641555483Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1871}
226812023-09-22T23:34:36.786ZRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
226822023-09-22T23:34:36.790ZLoading write count information from file "/tmp/test_repair_verify.data"
226832023-09-22T23:34:36.793ZRun Repair workload
226842023-09-22T23:34:36.796Z01/30 Write block 536 len 4096 data: 15
226852023-09-22T23:34:36.799Z02/30 Flush
226862023-09-22T23:34:36.803Z03/30 Write block 462 len 20480 data: 14 14 13 17 15
226872023-09-22T23:34:36.806Z04/30 Read block 471 len 40960
226882023-09-22T23:34:36.812Z05/30 Read block 544 len 8192
226892023-09-22T23:34:36.816Z06/30 Write block 207 len 8192 data: 20 23
226902023-09-22T23:34:36.819Z07/30 Write block 279 len 16384 data: 22 22 20 20
226912023-09-22T23:34:36.823Z08/30 Write block 383 len 36864 data: 10 10 10 8 13 14 14 14 11
226922023-09-22T23:34:36.827Z09/30 Read block 206 len 16384
226932023-09-22T23:34:36.831Z10/30 Write block 208 len 8192 data: 24 19
226942023-09-22T23:34:36.834Z11/30 Write block 322 len 20480 data: 9 12 10 7 9
226952023-09-22T23:34:36.838Z12/30 Read block 120 len 12288
226962023-09-22T23:34:36.842Z13/30 Read block 222 len 20480
226972023-09-22T23:34:36.845Z14/30 Read block 552 len 8192
226982023-09-22T23:34:36.848Z15/30 Read block 518 len 24576
226992023-09-22T23:34:36.851Z16/30 Read block 289 len 40960
227002023-09-22T23:34:36.854Z17/30 Write block 285 len 24576 data: 14 14 14 11 9 12
227012023-09-22T23:34:36.859Z18/30 Read block 516 len 28672
227022023-09-22T23:34:36.862Z19/30 Write block 524 len 32768 data: 14 16 15 17 15 14 12 13
227032023-09-22T23:34:36.866Z20/30 Read block 122 len 12288
227042023-09-22T23:34:36.869Z21/30 Read block 557 len 4096
227052023-09-22T23:34:36.872Z22/30 Write block 202 len 8192 data: 15 18
227062023-09-22T23:34:36.877Z23/30 Read block 431 len 40960
227072023-09-22T23:34:36.880Z24/30 Flush
227082023-09-22T23:34:36.883Z25/30 Write block 383 len 20480 data: 11 11 11 9 14
227092023-09-22T23:34:36.886Z26/30 Read block 92 len 32768
227102023-09-22T23:34:36.889Z27/30 Read block 496 len 32768
227112023-09-22T23:34:36.892Z28/30 Read block 388 len 20480
227122023-09-22T23:34:36.896Z29/30 Read block 583 len 20480
227132023-09-22T23:34:36.900Z30/30 Read block 535 len 20480
227142023-09-22T23:34:36.903Z----------------------------------------------------------------
227152023-09-22T23:34:36.906Z Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:28
227162023-09-22T23:34:36.909ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
227172023-09-22T23:34:36.914Z 3 Acked 1002 Write 5 Done Done Done false
227182023-09-22T23:34:36.917Z 4 Acked 1003 Read 10 Done Done Done false
227192023-09-22T23:34:36.920Z 5 Acked 1004 Read 2 Done Done Sent false
227202023-09-22T23:34:36.924Z 6 Acked 1005 Write 2 Done Done Sent false
227212023-09-22T23:34:36.927Z 7 Acked 1006 Write 4 Done Done Sent false
227222023-09-22T23:34:36.930Z 8 Acked 1007 Write 9 Done Done Sent false
227232023-09-22T23:34:36.934Z 9 Acked 1008 Read 4 Done Done Sent false
227242023-09-22T23:34:36.937Z 10 Acked 1009 Write 2 Done Done Sent false
227252023-09-22T23:34:36.940Z 11 Acked 1010 Write 5 Done Done Sent false
227262023-09-22T23:34:36.943Z 12 Acked 1011 Read 3 Done Done Sent false
227272023-09-22T23:34:36.947Z 13 Acked 1012 Read 5 Done Done Sent false
227282023-09-22T23:34:36.950Z 14 Acked 1013 Read 2 Done Done Sent false
227292023-09-22T23:34:36.953Z 15 Acked 1014 Read 6 Done Done Sent false
227302023-09-22T23:34:36.956Z 16 Acked 1015 Read 10 Done Done Sent false
227312023-09-22T23:34:36.959Z 17 Acked 1016 Write 6 Done Done Sent false
227322023-09-22T23:34:36.964Z 18 Acked 1017 Read 7 Done Done Sent false
227332023-09-22T23:34:36.967Z 19 Acked 1018 Write 8 Done Done Sent false
227342023-09-22T23:34:36.970Z 20 Acked 1019 Read 3 Done Done Sent false
227352023-09-22T23:34:36.973Z 21 Acked 1020 Read 1 Done Done Sent false
227362023-09-22T23:34:36.976Z 22 Acked 1021 Write 2 Done Done Sent false
227372023-09-22T23:34:36.980Z 23 Acked 1022 Read 10 Done Done Sent false
227382023-09-22T23:34:36.983Z 24 Acked 1023 Flush 0 Done Done Sent false
227392023-09-22T23:34:36.986Z 25 Acked 1024 Write 5 Done Done Sent false
227402023-09-22T23:34:36.990Z 26 Acked 1025 Read 8 Done Done Sent false
227412023-09-22T23:34:36.993Z 27 Acked 1026 Read 8 Done Done Sent false
227422023-09-22T23:34:36.996Z 28 Acked 1027 Read 5 Done Done Sent false
227432023-09-22T23:34:37.001Z 29 Acked 1028 Read 5 Done Done Sent false
227442023-09-22T23:34:37.004Z 30 Acked 1029 Read 5 Done Sent Sent false
227452023-09-22T23:34:37.007Z STATES DS:0 DS:1 DS:2 TOTAL
227462023-09-22T23:34:37.010Z New 0 0 0 0
227472023-09-22T23:34:37.013Z Sent 0 1 26 27
227482023-09-22T23:34:37.016Z Done 28 27 2 57
227492023-09-22T23:34:37.019Z Skipped 0 0 0 0
227502023-09-22T23:34:37.022Z Error 0 0 0 0
227512023-09-22T23:34:37.025ZLast Flush: 1023 1023 1001
227522023-09-22T23:34:37.028ZDownstairs last five completed: 1001 1000
227532023-09-22T23:34:37.032ZUpstairs last five completed: 30 29 28 27 26 25
227542023-09-22T23:34:37.036ZWrote out file "/tmp/test_repair_verify.data"
227552023-09-22T23:34:37.039Z
227562023-09-22T23:34:37.042ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
227572023-09-22T23:34:37.046Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.768915402Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1872}
227582023-09-22T23:34:37.051ZError: Difference in extent metadata found!
227592023-09-22T23:34:37.055Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.769259315Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1872}
227602023-09-22T23:34:37.058Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.769282938Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1872}
227612023-09-22T23:34:37.061Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.79125283Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1872}
227622023-09-22T23:34:37.064Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.791284561Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1872}
227632023-09-22T23:34:37.067Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.791303456Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1872}
227642023-09-22T23:34:37.074Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.803580275Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1872}
227652023-09-22T23:34:37.077Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.803610574Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1872}
227662023-09-22T23:34:37.080Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:36.803629371Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1872}
227672023-09-22T23:34:37.083ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
227682023-09-22T23:34:37.086Z 0 000-019 169 169 169 273 273 273 F F F
227692023-09-22T23:34:37.091Z 1 020-039 168 168 168 270 270 270 F F F
227702023-09-22T23:34:37.095Z 2 040-059 170 170 170 275 275 275 F F F
227712023-09-22T23:34:37.098Z 3 060-079 171 171 171 276 276 276 F F F
227722023-09-22T23:34:37.101Z 4 080-099 171 171 171 276 276 276 F F F
227732023-09-22T23:34:37.105Z 5 100-119 163 163 163 260 260 260 F F F
227742023-09-22T23:34:37.108Z 6 120-139 168 168 168 270 270 270 F F F
227752023-09-22T23:34:37.113Z 7 140-159 166 166 166 267 267 267 F F F
227762023-09-22T23:34:37.116Z 8 160-179 168 168 168 272 272 272 F F F
227772023-09-22T23:34:37.119Z 9 180-199 164 164 164 263 263 263 F F F
227782023-09-22T23:34:37.122Z 10 200-219 172 172 170 278 278 275 F F F <---
227792023-09-22T23:34:37.125Z 11 220-239 170 170 170 275 275 275 F F F
227802023-09-22T23:34:37.128Z 12 240-259 170 170 170 274 274 274 F F F
227812023-09-22T23:34:37.132Z 13 260-279 172 172 168 278 278 271 F F F <---
227822023-09-22T23:34:37.135Z 14 280-299 172 172 168 278 278 270 F F F <---
227832023-09-22T23:34:37.138Z 15 300-319 165 165 165 264 264 264 F F F
227842023-09-22T23:34:37.141Z 16 320-339 172 172 168 278 278 270 F F F <---
227852023-09-22T23:34:37.145Z 17 340-359 171 171 171 276 276 276 F F F
227862023-09-22T23:34:37.148Z 18 360-379 171 171 171 276 276 276 F F F
227872023-09-22T23:34:37.151Z 19 380-399 172 172 169 278 278 273 T T F <---
227882023-09-22T23:34:37.155Z 20 400-419 169 169 169 273 273 273 F F F
227892023-09-22T23:34:37.158Z 21 420-439 171 171 171 276 276 276 F F F
227902023-09-22T23:34:37.161Z 22 440-459 165 165 165 264 264 264 F F F
227912023-09-22T23:34:37.165Z 23 460-479 172 172 171 278 278 276 F F T <---
227922023-09-22T23:34:37.171Z 24 480-499 170 170 170 275 275 275 F F F
227932023-09-22T23:34:37.174Z 25 500-519 171 171 171 276 276 276 F F F
227942023-09-22T23:34:37.178Z 26 520-539 172 172 172 278 278 277 F F F <---
227952023-09-22T23:34:37.181Z 27 540-559 169 169 169 273 273 273 F F F
227962023-09-22T23:34:37.184Z 28 560-579 168 168 168 270 270 270 F F F
227972023-09-22T23:34:37.190Z 29 580-599 166 166 166 265 265 265 F F F
227982023-09-22T23:34:37.193ZMax gen: 172, Max flush: 278
227992023-09-22T23:34:37.196ZA Difference in extent metadata is expected here
228002023-09-22T23:34:37.199ZOn loop 86
228012023-09-22T23:34:37.202Z
228022023-09-22T23:34:37.208ZVerifying data now
228032023-09-22T23:34:42.735ZNo change after verify
228042023-09-22T23:34:42.738ZLoop: 86 Downstairs dump after verify (and repair):
228052023-09-22T23:34:42.753Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.750855894Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1877}
228062023-09-22T23:34:42.756Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.751280059Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1877}
228072023-09-22T23:34:42.759Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.751305455Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1877}
228082023-09-22T23:34:42.771Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.769751609Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1877}
228092023-09-22T23:34:42.774Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.769800945Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1877}
228102023-09-22T23:34:42.777Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.769821212Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1877}
228112023-09-22T23:34:42.782Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.780550691Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1877}
228122023-09-22T23:34:42.785Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.780603549Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1877}
228132023-09-22T23:34:42.788Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.780624263Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1877}
228142023-09-22T23:34:42.793ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
228152023-09-22T23:34:42.796Z 0 000-019 169 169 169 273 273 273 F F F
228162023-09-22T23:34:42.799Z 1 020-039 168 168 168 270 270 270 F F F
228172023-09-22T23:34:42.802Z 2 040-059 170 170 170 275 275 275 F F F
228182023-09-22T23:34:42.804Z 3 060-079 171 171 171 276 276 276 F F F
228192023-09-22T23:34:42.807Z 4 080-099 171 171 171 276 276 276 F F F
228202023-09-22T23:34:42.810Z 5 100-119 163 163 163 260 260 260 F F F
228212023-09-22T23:34:42.813Z 6 120-139 168 168 168 270 270 270 F F F
228222023-09-22T23:34:42.816Z 7 140-159 166 166 166 267 267 267 F F F
228232023-09-22T23:34:42.819Z 8 160-179 168 168 168 272 272 272 F F F
228242023-09-22T23:34:42.822Z 9 180-199 164 164 164 263 263 263 F F F
228252023-09-22T23:34:42.825Z 10 200-219 172 172 172 278 278 278 F F F
228262023-09-22T23:34:42.828Z 11 220-239 170 170 170 275 275 275 F F F
228272023-09-22T23:34:42.831Z 12 240-259 170 170 170 274 274 274 F F F
228282023-09-22T23:34:42.834Z 13 260-279 172 172 172 278 278 278 F F F
228292023-09-22T23:34:42.837Z 14 280-299 172 172 172 278 278 278 F F F
228302023-09-22T23:34:42.840Z 15 300-319 165 165 165 264 264 264 F F F
228312023-09-22T23:34:42.842Z 16 320-339 172 172 172 278 278 278 F F F
228322023-09-22T23:34:42.845Z 17 340-359 171 171 171 276 276 276 F F F
228332023-09-22T23:34:42.848Z 18 360-379 171 171 171 276 276 276 F F F
228342023-09-22T23:34:42.851Z 19 380-399 173 173 173 279 279 279 F F F
228352023-09-22T23:34:42.854Z 20 400-419 169 169 169 273 273 273 F F F
228362023-09-22T23:34:42.856Z 21 420-439 171 171 171 276 276 276 F F F
228372023-09-22T23:34:42.859Z 22 440-459 165 165 165 264 264 264 F F F
228382023-09-22T23:34:42.862Z 23 460-479 172 172 172 278 278 278 F F F
228392023-09-22T23:34:42.864Z 24 480-499 170 170 170 275 275 275 F F F
228402023-09-22T23:34:42.867Z 25 500-519 171 171 171 276 276 276 F F F
228412023-09-22T23:34:42.869Z 26 520-539 172 172 172 278 278 278 F F F
228422023-09-22T23:34:42.872Z 27 540-559 169 169 169 273 273 273 F F F
228432023-09-22T23:34:42.874Z 28 560-579 168 168 168 270 270 270 F F F
228442023-09-22T23:34:42.877Z 29 580-599 166 166 166 265 265 265 F F F
228452023-09-22T23:34:42.880ZMax gen: 173, Max flush: 279
228462023-09-22T23:34:42.882Z
228472023-09-22T23:34:42.885ZBegin loop 87
228482023-09-22T23:34:42.887ZDownstairs to restart: 2
228492023-09-22T23:34:42.890Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.828303271Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228502023-09-22T23:34:42.893Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.828749435Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228512023-09-22T23:34:42.895Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.82888859Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228522023-09-22T23:34:42.898Z{"msg":"Crucible stats registered with UUID: 46c4ed52-ce4e-459e-98ce-d506f937a6ec","v":0,"name":"crucible","level":30Crucible runtime is spawned
228532023-09-22T23:34:42.901Z,"time":"2023-09-22T23:34:42.82892851Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228542023-09-22T23:34:42.904Z{"msg":"The guest has requested activation with gen:174
228552023-09-22T23:34:42.906ZCrucible 46c4ed52-ce4e-459e-98ce-d506f937a6ec has session id: 66382114-052d-4f10-9697-8ec4c25a5e88","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.829125979Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228562023-09-22T23:34:42.909Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.829199587Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879,"task":"up_listen"}
228572023-09-22T23:34:42.912Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.829248445Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228582023-09-22T23:34:42.915Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.829287755Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228592023-09-22T23:34:42.917Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.829308974Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879,"looper":"0"}
228602023-09-22T23:34:42.920Z{"msg":"[0] 46c4ed52-ce4e-459e-98ce-d506f937a6ec looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.82961695Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879,"looper":"0"}
228612023-09-22T23:34:42.923Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.829741334Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879,"looper":"1"}
228622023-09-22T23:34:42.925Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.82986941Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879,"looper":"2"}
228632023-09-22T23:34:42.928Z{"msg":"[1] 46c4ed52-ce4e-459e-98ce-d506f937a6ec looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.829951655Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879,"looper":"1"}
228642023-09-22T23:34:42.931Z{"msg":"46c4ed52-ce4e-459e-98ce-d506f937a6ec active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.830288315Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228652023-09-22T23:34:42.934Z{"msg":"Set desired generation to :174","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.830362876Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228662023-09-22T23:34:42.937Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.830564995Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228672023-09-22T23:34:42.941Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.831006914Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228682023-09-22T23:34:42.943Z{"msg":"[0] 46c4ed52-ce4e-459e-98ce-d506f937a6ec (99850c31-13fa-43cd-9285-108154eeeb72) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.831200471Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228692023-09-22T23:34:42.946Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.831294839Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228702023-09-22T23:34:42.949Z{"msg":"[0] client is_active_req TRUE, promote! session 99850c31-13fa-43cd-9285-108154eeeb72","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.831339339Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228712023-09-22T23:34:42.952Z{"msg":"[1] 46c4ed52-ce4e-459e-98ce-d506f937a6ec (99850c31-13fa-43cd-9285-108154eeeb72) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.831381247Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228722023-09-22T23:34:42.954Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.831401777Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228732023-09-22T23:34:42.957Z{"msg":"[1] client is_active_req TRUE, promote! session 99850c31-13fa-43cd-9285-108154eeeb72","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.831427904Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228742023-09-22T23:34:42.960Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.831917406Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228752023-09-22T23:34:42.963Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.831983205Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228762023-09-22T23:34:42.965Z{"msg":"46c4ed52-ce4e-459e-98ce-d506f937a6ec WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.832084323Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228772023-09-22T23:34:42.968Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.832150852Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228782023-09-22T23:34:42.971Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.832203054Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228792023-09-22T23:34:42.974Z{"msg":"46c4ed52-ce4e-459e-98ce-d506f937a6ec WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.832243778Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228802023-09-22T23:34:42.977Z{"msg":"[1] 46c4ed52-ce4e-459e-98ce-d506f937a6ec (99850c31-13fa-43cd-9285-108154eeeb72) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.834329114Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228812023-09-22T23:34:42.980Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.834389375Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228822023-09-22T23:34:42.982Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:34:42.834431864Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228832023-09-22T23:34:42.985Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.834470424Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228842023-09-22T23:34:42.988Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.834509342Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228852023-09-22T23:34:42.991Z{"msg":"46c4ed52-ce4e-459e-98ce-d506f937a6ec WaitActive WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.834547927Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228862023-09-22T23:34:42.994Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.834585625Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228872023-09-22T23:34:42.997Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.83462309Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228882023-09-22T23:34:43.000Z{"msg":"[0] 46c4ed52-ce4e-459e-98ce-d506f937a6ec (99850c31-13fa-43cd-9285-108154eeeb72) WaitActive WaitQuorum New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.835103874Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228892023-09-22T23:34:43.004Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.835185652Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228902023-09-22T23:34:43.007Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:34:42.835234457Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228912023-09-22T23:34:43.010Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.835273591Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228922023-09-22T23:34:43.013Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.835312455Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228932023-09-22T23:34:43.015Z{"msg":"46c4ed52-ce4e-459e-98ce-d506f937a6ec WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.835351339Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228942023-09-22T23:34:43.018Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.835389132Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228952023-09-22T23:34:43.021Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:42.835413594Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228962023-09-22T23:34:43.834Z{"msg":"[2] 46c4ed52-ce4e-459e-98ce-d506f937a6ec looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.831650694Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879,"looper":"2"}
228972023-09-22T23:34:43.837Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.831733768Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228982023-09-22T23:34:43.840Z{"msg":"[2] 46c4ed52-ce4e-459e-98ce-d506f937a6ec (99850c31-13fa-43cd-9285-108154eeeb72) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.832926057Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
228992023-09-22T23:34:43.843Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.832980778Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229002023-09-22T23:34:43.846Z{"msg":"[2] client is_active_req TRUE, promote! session 99850c31-13fa-43cd-9285-108154eeeb72","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.833024734Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229012023-09-22T23:34:43.849Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.833627976Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229022023-09-22T23:34:43.852Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.833745719Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229032023-09-22T23:34:43.855Z{"msg":"46c4ed52-ce4e-459e-98ce-d506f937a6ec WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.833812625Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229042023-09-22T23:34:43.858Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.835929091Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229052023-09-22T23:34:43.862Z{"msg":"[2] 46c4ed52-ce4e-459e-98ce-d506f937a6ec (99850c31-13fa-43cd-9285-108154eeeb72) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.83621238Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229062023-09-22T23:34:43.865Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.836282305Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229072023-09-22T23:34:43.868Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:34:43.836319993Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229082023-09-22T23:34:43.871Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.83637316Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229092023-09-22T23:34:43.875Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.836415468Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229102023-09-22T23:34:43.877Z{"msg":"46c4ed52-ce4e-459e-98ce-d506f937a6ec WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.836455239Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229112023-09-22T23:34:43.881Z{"msg":"[0]R flush_numbers[0..12]: [273, 270, 275, 276, 276, 260, 270, 267, 272, 263, 278, 275]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.836495939Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229122023-09-22T23:34:43.884Z{"msg":"[0]R generation[0..12]: [169, 168, 170, 171, 171, 163, 168, 166, 168, 164, 172, 170]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.836535726Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229132023-09-22T23:34:43.887Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.836575936Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229142023-09-22T23:34:43.890Z{"msg":"[1]R flush_numbers[0..12]: [273, 270, 275, 276, 276, 260, 270, 267, 272, 263, 278, 275]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.836615646Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229152023-09-22T23:34:43.893Z{"msg":"[1]R generation[0..12]: [169, 168, 170, 171, 171, 163, 168, 166, 168, 164, 172, 170]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.836655425Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229162023-09-22T23:34:43.896Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.836677179Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229172023-09-22T23:34:43.899Z{"msg":"[2]R flush_numbers[0..12]: [273, 270, 275, 276, 276, 260, 270, 267, 272, 263, 278, 275]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.836706861Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229182023-09-22T23:34:43.902ZThe guest has finished waiting for activation with:174
229192023-09-22T23:34:43.905Z{"msg":"Wait for a query_work_queue command to finish before sending IO
229202023-09-22T23:34:43.908Z[2]R generation[0..12]: [169, 168, 170, 171, 171, 163, 168, 166, 168, 164, 172, 170]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.836751498Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229212023-09-22T23:34:43.911Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.83679321Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229222023-09-22T23:34:43.914Z{"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.836812758Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229232023-09-22T23:34:43.917Z{"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.836830768Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229242023-09-22T23:34:43.921Z{"msg":"Next flush: 280","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.836890177Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229252023-09-22T23:34:43.923Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.836942087Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229262023-09-22T23:34:43.926Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.836993416Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229272023-09-22T23:34:43.929Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.837063428Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229282023-09-22T23:34:43.932Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.837121609Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229292023-09-22T23:34:43.934Z{"msg":"46c4ed52-ce4e-459e-98ce-d506f937a6ec is now active with session: 99850c31-13fa-43cd-9285-108154eeeb72","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.837186285Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229302023-09-22T23:34:43.937Z{"msg":"46c4ed52-ce4e-459e-98ce-d506f937a6ec Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.837244839Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229312023-09-22T23:34:43.940Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.837306826Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229322023-09-22T23:34:43.942Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.837362567Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229332023-09-22T23:34:43.945Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.837421874Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229342023-09-22T23:34:43.947Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.837475137Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229352023-09-22T23:34:43.950Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
229362023-09-22T23:34:43.953Z,"time":"2023-09-22T23:34:43.837554124Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229372023-09-22T23:34:43.956Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.837678536Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229382023-09-22T23:34:43.959Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.837734229Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229392023-09-22T23:34:43.962Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.837787446Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229402023-09-22T23:34:43.968Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.837841514Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229412023-09-22T23:34:43.971Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.837894165Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229422023-09-22T23:34:43.974Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.837963377Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1879}
229432023-09-22T23:34:43.977ZLoading write count information from file "/tmp/test_repair_verify.data"
229442023-09-22T23:34:43.981ZRun Repair workload
229452023-09-22T23:34:43.984Z01/30 Write block 254 len 4096 data: 11
229462023-09-22T23:34:43.987Z02/30 Read block 370 len 40960
229472023-09-22T23:34:43.989Z03/30 Flush
229482023-09-22T23:34:43.992Z04/30 Write block 176 len 16384 data: 20 18 17 14
229492023-09-22T23:34:43.995Z05/30 Write block 314 len 28672 data: 12 9 10 10 9 9 12
229502023-09-22T23:34:43.997Z06/30 Read block 87 len 20480
229512023-09-22T23:34:44.001Z07/30 Read block 33 len 24576
229522023-09-22T23:34:44.004Z08/30 Read block 369 len 8192
229532023-09-22T23:34:44.007Z09/30 Write block 53 len 20480 data: 7 5 6 7 10
229542023-09-22T23:34:44.010Z10/30 Read block 63 len 32768
229552023-09-22T23:34:44.013Z11/30 Write block 285 len 36864 data: 15 15 15 12 10 13 15 22 18
229562023-09-22T23:34:44.016Z12/30 Write block 85 len 4096 data: 11
229572023-09-22T23:34:44.019Z13/30 Read block 348 len 16384
229582023-09-22T23:34:44.022Z14/30 Read block 71 len 32768
229592023-09-22T23:34:44.025Z15/30 Read block 475 len 36864
229602023-09-22T23:34:44.028Z16/30 Write block 510 len 28672 data: 14 12 15 17 20 19 17
229612023-09-22T23:34:44.031Z17/30 Read block 376 len 20480
229622023-09-22T23:34:44.034Z18/30 Flush
229632023-09-22T23:34:44.036Z19/30 Write block 318 len 16384 data: 10 10 13 12
229642023-09-22T23:34:44.039Z20/30 Read block 558 len 40960
229652023-09-22T23:34:44.042Z21/30 Read block 199 len 12288
229662023-09-22T23:34:44.045Z22/30 Write block 526 len 20480 data: 16 18 16 15 13
229672023-09-22T23:34:44.048Z23/30 Flush
229682023-09-22T23:34:44.050Z24/30 Write block 444 len 24576 data: 18 17 10 8 11 12
229692023-09-22T23:34:44.053Z25/30 Write block 359 len 4096 data: 18
229702023-09-22T23:34:44.056Z26/30 Write block 348 len 24576 data: 13 12 12 14 14 14
229712023-09-22T23:34:44.059Z27/30 Write block 491 len 28672 data: 24 20 19 17 19 19 20
229722023-09-22T23:34:44.062Z28/30 Read block 490 len 16384
229732023-09-22T23:34:44.065Z29/30 Read block 187 len 40960
229742023-09-22T23:34:44.068Z30/30 Read block 45 len 16384
229752023-09-22T23:34:44.071Z----------------------------------------------------------------
229762023-09-22T23:34:44.073Z Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:27
229772023-09-22T23:34:44.076ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
229782023-09-22T23:34:44.079Z 4 Acked 1003 Write 4 Done Done Sent false
229792023-09-22T23:34:44.082Z 5 Acked 1004 Write 7 Done Done Sent false
229802023-09-22T23:34:44.084Z 6 Acked 1005 Read 5 Done Done Sent false
229812023-09-22T23:34:44.087Z 7 Acked 1006 Read 6 Done Done Sent false
229822023-09-22T23:34:44.090Z 8 Acked 1007 Read 2 Done Done Sent false
229832023-09-22T23:34:44.093Z 9 Acked 1008 Write 5 Done Done Sent false
229842023-09-22T23:34:44.097Z 10 Acked 1009 Read 8 Done Done Sent false
229852023-09-22T23:34:44.100Z 11 Acked 1010 Write 9 Done Done Sent false
229862023-09-22T23:34:44.103Z 12 Acked 1011 Write 1 Done Done Sent false
229872023-09-22T23:34:44.105Z 13 Acked 1012 Read 4 Done Done Sent false
229882023-09-22T23:34:44.108Z 14 Acked 1013 Read 8 Done Done Sent false
229892023-09-22T23:34:44.111Z 15 Acked 1014 Read 9 Done Done Sent false
229902023-09-22T23:34:44.114Z 16 Acked 1015 Write 7 Done Done Sent false
229912023-09-22T23:34:44.117Z 17 Acked 1016 Read 5 Done Done Sent false
229922023-09-22T23:34:44.120Z 18 Acked 1017 Flush 0 Done Done Sent false
229932023-09-22T23:34:44.122Z 19 Acked 1018 Write 4 Done Done Sent false
229942023-09-22T23:34:44.125Z 20 Acked 1019 Read 10 Done Done Sent false
229952023-09-22T23:34:44.128Z 21 Acked 1020 Read 3 Done Done Sent false
229962023-09-22T23:34:44.131Z 22 Acked 1021 Write 5 Done Done Sent false
229972023-09-22T23:34:44.134Z 23 Acked 1022 Flush 0 Done Done Sent false
229982023-09-22T23:34:44.137Z 24 Acked 1023 Write 6 Done Done Sent false
229992023-09-22T23:34:44.139Z 25 Acked 1024 Write 1 Done Done Sent false
230002023-09-22T23:34:44.142Z 26 Acked 1025 Write 6 Done Done Sent false
230012023-09-22T23:34:44.145Z 27 Acked 1026 Write 7 Done Done Sent false
230022023-09-22T23:34:44.148Z 28 Acked 1027 Read 4 Done Done Sent false
230032023-09-22T23:34:44.151Z 29 Acked 1028 Read 10 Done Done Sent false
230042023-09-22T23:34:44.154Z 30 Acked 1029 Read 4 Done Done Sent false
230052023-09-22T23:34:44.157Z STATES DS:0 DS:1 DS:2 TOTAL
230062023-09-22T23:34:44.159Z New 0 0 0 0
230072023-09-22T23:34:44.162Z Sent 0 0 27 27
230082023-09-22T23:34:44.165Z Done 27 27 0 54
230092023-09-22T23:34:44.168Z Skipped 0 0 0 0
230102023-09-22T23:34:44.171Z Error 0 0 0 0
230112023-09-22T23:34:44.174ZLast Flush: 1022 1022 1002
230122023-09-22T23:34:44.177ZDownstairs last five completed: 1002 1001 1000
230132023-09-22T23:34:44.179ZUpstairs last five completed: 30 29 28 27 26 25
230142023-09-22T23:34:44.182ZWrote out file "/tmp/test_repair_verify.data"
230152023-09-22T23:34:44.185Z
230162023-09-22T23:34:44.188ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
230172023-09-22T23:34:44.190Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.980574598Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1880}
230182023-09-22T23:34:44.193Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.980899248Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1880}
230192023-09-22T23:34:44.196Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:43.980922305Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1880}
230202023-09-22T23:34:44.199Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:44.004682098Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1880}
230212023-09-22T23:34:44.202Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:44.004714516Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1880}
230222023-09-22T23:34:44.205Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:44.004733738Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1880}
230232023-09-22T23:34:44.208Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:44.019590474Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1880}
230242023-09-22T23:34:44.211Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:44.019620929Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1880}
230252023-09-22T23:34:44.214Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:44.01963982Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1880}
230262023-09-22T23:34:44.217ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
230272023-09-22T23:34:44.219Z 0 000-019 169 169 169 273 273 273 F F F
230282023-09-22T23:34:44.222Z 1 020-039 168 168 168 270 270 270 F F F
230292023-09-22T23:34:44.225ZError: Difference in extent metadata found!
230302023-09-22T23:34:44.228Z 2 040-059 174 174 170 281 281 275 F F F <---
230312023-09-22T23:34:44.231Z 3 060-079 171 171 171 276 276 276 F F F
230322023-09-22T23:34:44.234Z 4 080-099 174 174 171 281 281 276 F F F <---
230332023-09-22T23:34:44.237Z 5 100-119 163 163 163 260 260 260 F F F
230342023-09-22T23:34:44.239Z 6 120-139 168 168 168 270 270 270 F F F
230352023-09-22T23:34:44.242Z 7 140-159 166 166 166 267 267 267 F F F
230362023-09-22T23:34:44.244Z 8 160-179 174 174 168 281 281 272 F F F <---
230372023-09-22T23:34:44.247Z 9 180-199 164 164 164 263 263 263 F F F
230382023-09-22T23:34:44.250Z 10 200-219 172 172 172 278 278 278 F F F
230392023-09-22T23:34:44.253Z 11 220-239 170 170 170 275 275 275 F F F
230402023-09-22T23:34:44.256Z 12 240-259 174 174 174 280 280 280 F F F
230412023-09-22T23:34:44.259Z 13 260-279 172 172 172 278 278 278 F F F
230422023-09-22T23:34:44.261Z 14 280-299 174 174 172 281 281 278 F F F <---
230432023-09-22T23:34:44.264Z 15 300-319 174 174 165 282 282 264 F F F <---
230442023-09-22T23:34:44.267Z 16 320-339 174 174 172 282 282 278 F F F <---
230452023-09-22T23:34:44.270Z 17 340-359 171 171 171 276 276 276 T T F <---
230462023-09-22T23:34:44.274Z 18 360-379 171 171 171 276 276 276 F F F
230472023-09-22T23:34:44.277Z 19 380-399 173 173 173 279 279 279 F F F
230482023-09-22T23:34:44.280Z 20 400-419 169 169 169 273 273 273 F F F
230492023-09-22T23:34:44.283Z 21 420-439 171 171 171 276 276 276 F F F
230502023-09-22T23:34:44.286Z 22 440-459 165 165 165 264 264 264 T T F <---
230512023-09-22T23:34:44.289Z 23 460-479 172 172 172 278 278 278 F F F
230522023-09-22T23:34:44.292Z 24 480-499 170 170 170 275 275 275 T T F <---
230532023-09-22T23:34:44.295Z 25 500-519 174 174 171 281 281 276 F F F <---
230542023-09-22T23:34:44.298Z 26 520-539 174 174 172 282 282 278 F F F <---
230552023-09-22T23:34:44.300Z 27 540-559 169 169 169 273 273 273 F F F
230562023-09-22T23:34:44.303Z 28 560-579 168 168 168 270 270 270 F F F
230572023-09-22T23:34:44.306Z 29 580-599 166 166 166 265 265 265 F F F
230582023-09-22T23:34:44.309ZMax gen: 174, Max flush: 282
230592023-09-22T23:34:44.312ZA Difference in extent metadata is expected here
230602023-09-22T23:34:44.315ZOn loop 87
230612023-09-22T23:34:44.318Z
230622023-09-22T23:34:44.321ZVerifying data now
230632023-09-22T23:34:50.280ZNo change after verify
230642023-09-22T23:34:50.283ZLoop: 87 Downstairs dump after verify (and repair):
230652023-09-22T23:34:50.298Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.296022715Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1885}
230662023-09-22T23:34:50.301Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.296444779Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1885}
230672023-09-22T23:34:50.304Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.296470125Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1885}
230682023-09-22T23:34:50.316Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.315135149Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1885}
230692023-09-22T23:34:50.319Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.315188675Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1885}
230702023-09-22T23:34:50.322Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.315209056Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1885}
230712023-09-22T23:34:50.327Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.326011666Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1885}
230722023-09-22T23:34:50.330Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.326062814Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1885}
230732023-09-22T23:34:50.333Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.326083159Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1885}
230742023-09-22T23:34:50.338ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
230752023-09-22T23:34:50.342Z 0 000-019 169 169 169 273 273 273 F F F
230762023-09-22T23:34:50.345Z 1 020-039 168 168 168 270 270 270 F F F
230772023-09-22T23:34:50.348Z 2 040-059 174 174 174 281 281 281 F F F
230782023-09-22T23:34:50.351Z 3 060-079 171 171 171 276 276 276 F F F
230792023-09-22T23:34:50.354Z 4 080-099 174 174 174 281 281 281 F F F
230802023-09-22T23:34:50.357Z 5 100-119 163 163 163 260 260 260 F F F
230812023-09-22T23:34:50.360Z 6 120-139 168 168 168 270 270 270 F F F
230822023-09-22T23:34:50.362Z 7 140-159 166 166 166 267 267 267 F F F
230832023-09-22T23:34:50.365Z 8 160-179 174 174 174 281 281 281 F F F
230842023-09-22T23:34:50.368Z 9 180-199 164 164 164 263 263 263 F F F
230852023-09-22T23:34:50.375Z 10 200-219 172 172 172 278 278 278 F F F
230862023-09-22T23:34:50.378Z 11 220-239 170 170 170 275 275 275 F F F
230872023-09-22T23:34:50.381Z 12 240-259 174 174 174 280 280 280 F F F
230882023-09-22T23:34:50.384Z 13 260-279 172 172 172 278 278 278 F F F
230892023-09-22T23:34:50.387Z 14 280-299 174 174 174 281 281 281 F F F
230902023-09-22T23:34:50.389Z 15 300-319 174 174 174 282 282 282 F F F
230912023-09-22T23:34:50.392Z 16 320-339 174 174 174 282 282 282 F F F
230922023-09-22T23:34:50.395Z 17 340-359 175 175 175 283 283 283 F F F
230932023-09-22T23:34:50.398Z 18 360-379 171 171 171 276 276 276 F F F
230942023-09-22T23:34:50.401Z 19 380-399 173 173 173 279 279 279 F F F
230952023-09-22T23:34:50.404Z 20 400-419 169 169 169 273 273 273 F F F
230962023-09-22T23:34:50.407Z 21 420-439 171 171 171 276 276 276 F F F
230972023-09-22T23:34:50.410Z 22 440-459 175 175 175 283 283 283 F F F
230982023-09-22T23:34:50.413Z 23 460-479 172 172 172 278 278 278 F F F
230992023-09-22T23:34:50.417Z 24 480-499 175 175 175 283 283 283 F F F
231002023-09-22T23:34:50.420Z 25 500-519 174 174 174 281 281 281 F F F
231012023-09-22T23:34:50.423Z 26 520-539 174 174 174 282 282 282 F F F
231022023-09-22T23:34:50.426Z 27 540-559 169 169 169 273 273 273 F F F
231032023-09-22T23:34:50.428Z 28 560-579 168 168 168 270 270 270 F F F
231042023-09-22T23:34:50.431Z 29 580-599 166 166 166 265 265 265 F F F
231052023-09-22T23:34:50.434ZMax gen: 175, Max flush: 283
231062023-09-22T23:34:50.439Z
231072023-09-22T23:34:50.442ZBegin loop 88
231082023-09-22T23:34:50.445ZDownstairs to restart: 1
231092023-09-22T23:34:50.448Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.379529728Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231102023-09-22T23:34:50.451Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.379969463Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231112023-09-22T23:34:50.457Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.380031779Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231122023-09-22T23:34:50.460Z{"msg":"Crucible stats registered with UUID: a5c01b07-7ed8-48a0-a229-b4c1f0809342","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.380093665Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231132023-09-22T23:34:50.462Z{"msg":"Crucible a5c01b07-7ed8-48a0-a229-b4c1f0809342 has session id: 148e4022-599d-42c1-b2df-5550eb97c502","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.380152689Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231142023-09-22T23:34:50.465ZCrucible runtime is spawned
231152023-09-22T23:34:50.468ZThe guest has requested activation with gen:176
231162023-09-22T23:34:50.471Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.38061549Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887,"task":"up_listen"}
231172023-09-22T23:34:50.474Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.380678643Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231182023-09-22T23:34:50.477Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.380721569Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887,"looper":"1"}
231192023-09-22T23:34:50.479Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.380772669Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231202023-09-22T23:34:50.483Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.380858124Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887,"looper":"0"}
231212023-09-22T23:34:50.485Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.38111263Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887,"looper":"2"}
231222023-09-22T23:34:50.488Z{"msg":"[0] a5c01b07-7ed8-48a0-a229-b4c1f0809342 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.381191015Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887,"looper":"0"}
231232023-09-22T23:34:50.491Z{"msg":"[2] a5c01b07-7ed8-48a0-a229-b4c1f0809342 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.381262393Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887,"looper":"2"}
231242023-09-22T23:34:50.494Z{"msg":"a5c01b07-7ed8-48a0-a229-b4c1f0809342 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.381505672Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231252023-09-22T23:34:50.497Z{"msg":"Set desired generation to :176","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.381555271Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231262023-09-22T23:34:50.500Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.381692088Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231272023-09-22T23:34:50.502Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.382009063Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231282023-09-22T23:34:50.505Z{"msg":"[0] a5c01b07-7ed8-48a0-a229-b4c1f0809342 (1339bd69-ff7c-46bb-9c7d-f7f0ec160095) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.38235137Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231292023-09-22T23:34:50.508Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.382423543Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231302023-09-22T23:34:50.511Z{"msg":"[0] client is_active_req TRUE, promote! session 1339bd69-ff7c-46bb-9c7d-f7f0ec160095","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.382522424Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231312023-09-22T23:34:50.514Z{"msg":"[2] a5c01b07-7ed8-48a0-a229-b4c1f0809342 (1339bd69-ff7c-46bb-9c7d-f7f0ec160095) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.382616895Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231322023-09-22T23:34:50.517Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.382670856Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231332023-09-22T23:34:50.519Z{"msg":"[2] client is_active_req TRUE, promote! session 1339bd69-ff7c-46bb-9c7d-f7f0ec160095","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.382714683Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231342023-09-22T23:34:51.525Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.383130567Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231352023-09-22T23:34:51.529Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.383192724Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231362023-09-22T23:34:51.532Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.383216981Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231372023-09-22T23:34:51.534Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.383279946Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231382023-09-22T23:34:51.537Z{"msg":"a5c01b07-7ed8-48a0-a229-b4c1f0809342 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.383323583Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231392023-09-22T23:34:51.540Z{"msg":"a5c01b07-7ed8-48a0-a229-b4c1f0809342 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.383384015Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231402023-09-22T23:34:51.543Z{"msg":"[0] a5c01b07-7ed8-48a0-a229-b4c1f0809342 (1339bd69-ff7c-46bb-9c7d-f7f0ec160095) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.385389932Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231412023-09-22T23:34:51.545Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.385506033Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231422023-09-22T23:34:51.548Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:34:50.385638901Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231432023-09-22T23:34:51.551Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.385723739Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231442023-09-22T23:34:51.554Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.38577072Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231452023-09-22T23:34:51.557Z{"msg":"a5c01b07-7ed8-48a0-a229-b4c1f0809342 WaitQuorum New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.385812674Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231462023-09-22T23:34:51.560Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.38585205Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231472023-09-22T23:34:51.563Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.385890352Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231482023-09-22T23:34:51.566Z{"msg":"[2] a5c01b07-7ed8-48a0-a229-b4c1f0809342 (1339bd69-ff7c-46bb-9c7d-f7f0ec160095) WaitQuorum New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.386458245Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231492023-09-22T23:34:51.568Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.38653222Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231502023-09-22T23:34:51.571Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:34:50.386580532Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231512023-09-22T23:34:51.574Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.386620367Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231522023-09-22T23:34:51.577Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.386659166Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231532023-09-22T23:34:51.580Z{"msg":"a5c01b07-7ed8-48a0-a229-b4c1f0809342 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.386698Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231542023-09-22T23:34:51.583Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.386736556Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231552023-09-22T23:34:51.586Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:50.386774092Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231562023-09-22T23:34:51.589Z{"msg":"[1] a5c01b07-7ed8-48a0-a229-b4c1f0809342 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.381881951Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887,"looper":"1"}
231572023-09-22T23:34:51.592Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.38194521Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231582023-09-22T23:34:51.595Z{"msg":"[1] a5c01b07-7ed8-48a0-a229-b4c1f0809342 (1339bd69-ff7c-46bb-9c7d-f7f0ec160095) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.383131043Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231592023-09-22T23:34:51.598Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.383189926Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231602023-09-22T23:34:51.600Z{"msg":"[1] client is_active_req TRUE, promote! session 1339bd69-ff7c-46bb-9c7d-f7f0ec160095","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.383233713Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231612023-09-22T23:34:51.603Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.383855851Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231622023-09-22T23:34:51.607Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.38391253Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231632023-09-22T23:34:51.609Z{"msg":"a5c01b07-7ed8-48a0-a229-b4c1f0809342 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.383955286Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231642023-09-22T23:34:51.612Z{"msg":"[1] a5c01b07-7ed8-48a0-a229-b4c1f0809342 (1339bd69-ff7c-46bb-9c7d-f7f0ec160095) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.386318922Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231652023-09-22T23:34:51.615Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.386374304Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231662023-09-22T23:34:51.618Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:34:51.386417538Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231672023-09-22T23:34:51.621Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.386456932Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231682023-09-22T23:34:51.624Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.386495553Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231692023-09-22T23:34:51.627Z{"msg":"a5c01b07-7ed8-48a0-a229-b4c1f0809342 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.386534869Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231702023-09-22T23:34:51.629Z{"msg":"[0]R flush_numbers[0..12]: [273, 270, 281, 276, 281, 260, 270, 267, 281, 263, 278, 275]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.386574972Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231712023-09-22T23:34:51.632Z{"msg":"[0]R generation[0..12]: [169, 168, 174, 171, 174, 163, 168, 166, 174, 164, 172, 170]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.386614746Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231722023-09-22T23:34:51.636Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.386655187Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231732023-09-22T23:34:51.639Z{"msg":"[1]R flush_numbers[0..12]: [273, 270, 281, 276, 281, 260, 270, 267, 281, 263, 278, 275]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.386695035Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231742023-09-22T23:34:51.642ZThe guest has finished waiting for activation with:176
231752023-09-22T23:34:51.645Z{"msg":"[1]R generation[0..12]: [169, 168, 174, 171, 174, 163, 168, 166, 174, 164, 172, 170]Wait for a query_work_queue command to finish before sending IO
231762023-09-22T23:34:51.648Z","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.386747222Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231772023-09-22T23:34:51.651Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.386788394Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231782023-09-22T23:34:51.654Z{"msg":"[2]R flush_numbers[0..12]: [273, 270, 281, 276, 281, 260, 270, 267, 281, 263, 278, 275]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.386828021Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231792023-09-22T23:34:51.657Z{"msg":"[2]R generation[0..12]: [169, 168, 174, 171, 174, 163, 168, 166, 174, 164, 172, 170]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.386929801Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231802023-09-22T23:34:51.660Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.386988662Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231812023-09-22T23:34:51.663Z{"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.387028087Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231822023-09-22T23:34:51.666Z{"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.387110303Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231832023-09-22T23:34:51.669Z{"msg":"Next flush: 284","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.387148384Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231842023-09-22T23:34:51.672Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.387189739Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231852023-09-22T23:34:51.675Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.387250913Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231862023-09-22T23:34:51.678Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.387270484Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231872023-09-22T23:34:51.681Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.387288223Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231882023-09-22T23:34:51.684Z{"msg":"a5c01b07-7ed8-48a0-a229-b4c1f0809342 is now active with session: 1339bd69-ff7c-46bb-9c7d-f7f0ec160095","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.387326767Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231892023-09-22T23:34:51.687Z{"msg":"a5c01b07-7ed8-48a0-a229-b4c1f0809342 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.387366207Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231902023-09-22T23:34:51.692Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.387406005Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231912023-09-22T23:34:51.695Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.387452475Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231922023-09-22T23:34:51.698Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.387490448Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231932023-09-22T23:34:51.702Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.387529714Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231942023-09-22T23:34:51.707Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.38756755Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231952023-09-22T23:34:51.710Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.387605604Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231962023-09-22T23:34:51.713Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.387651641Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231972023-09-22T23:34:51.718Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.387689393Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231982023-09-22T23:34:51.721Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.387735095Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
231992023-09-22T23:34:51.724Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.387777502Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
232002023-09-22T23:34:51.757Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
232012023-09-22T23:34:51.761Z30,"time":"2023-09-22T23:34:51.387823325Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1887}
232022023-09-22T23:34:51.767ZLoading write count information from file "/tmp/test_repair_verify.data"
232032023-09-22T23:34:51.771ZRun Repair workload
232042023-09-22T23:34:51.773Z01/30 Write block 27 len 32768 data: 13 10 8 8 10 7 7 11
232052023-09-22T23:34:51.777Z02/30 Read block 155 len 24576
232062023-09-22T23:34:51.781Z03/30 Write block 481 len 24576 data: 12 15 14 19 19 22
232072023-09-22T23:34:51.787Z04/30 Read block 537 len 16384
232082023-09-22T23:34:51.790Z05/30 Read block 167 len 28672
232092023-09-22T23:34:51.793Z06/30 Flush
232102023-09-22T23:34:51.800Z07/30 Write block 12 len 40960 data: 12 14 12 13 12 12 12 11 8 5
232112023-09-22T23:34:51.803Z08/30 Write block 327 len 32768 data: 9 13 13 10 13 14 13 17
232122023-09-22T23:34:51.806Z09/30 Write block 181 len 24576 data: 13 14 11 13 12 14
232132023-09-22T23:34:51.811Z10/30 Read block 99 len 28672
232142023-09-22T23:34:51.815Z11/30 Write block 51 len 36864 data: 12 9 8 6 7 8 11 13 14
232152023-09-22T23:34:51.818Z12/30 Flush
232162023-09-22T23:34:51.821Z13/30 Write block 63 len 32768 data: 23 23 25 24 23 25 22 15
232172023-09-22T23:34:51.824Z14/30 Read block 72 len 40960
232182023-09-22T23:34:51.827Z15/30 Read block 179 len 28672
232192023-09-22T23:34:51.830Z16/30 Flush
232202023-09-22T23:34:51.833Z17/30 Write block 143 len 20480 data: 13 8 10 10 10
232212023-09-22T23:34:51.836Z18/30 Write block 477 len 24576 data: 10 12 13 13 13 16
232222023-09-22T23:34:51.839Z19/30 Read block 548 len 8192
232232023-09-22T23:34:51.842Z20/30 Read block 464 len 20480
232242023-09-22T23:34:51.845Z21/30 Read block 560 len 20480
232252023-09-22T23:34:51.848Z22/30 Write block 538 len 4096 data: 18
232262023-09-22T23:34:51.851Z23/30 Write block 583 len 4096 data: 11
232272023-09-22T23:34:51.854Z24/30 Write block 350 len 28672 data: 13 15 15 15 14 14 13
232282023-09-22T23:34:51.857Z25/30 Read block 397 len 36864
232292023-09-22T23:34:51.860Z26/30 Write block 485 len 24576 data: 20 23 19 24 26 23
232302023-09-22T23:34:51.863ZError: Difference in extent metadata found!
232312023-09-22T23:34:51.866Z27/30 Write block 74 len 24576 data: 14 16 15 16 16 20
232322023-09-22T23:34:51.869Z28/30 Read block 321 len 24576
232332023-09-22T23:34:51.871Z29/30 Read block 487 len 20480
232342023-09-22T23:34:51.874Z30/30 Read block 566 len 28672
232352023-09-22T23:34:51.877Z----------------------------------------------------------------
232362023-09-22T23:34:51.879Z Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:14
232372023-09-22T23:34:51.882ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
232382023-09-22T23:34:51.885Z 17 Acked 1016 Write 5 Done Sent Done false
232392023-09-22T23:34:51.888Z 18 Acked 1017 Write 6 Done Sent Done false
232402023-09-22T23:34:51.891Z 19 Acked 1018 Read 2 Done Sent Done false
232412023-09-22T23:34:51.894Z 20 Acked 1019 Read 5 Done Sent Done false
232422023-09-22T23:34:51.897Z 21 Acked 1020 Read 5 Done Sent Done false
232432023-09-22T23:34:51.900Z 22 Acked 1021 Write 1 Done Sent Done false
232442023-09-22T23:34:51.902Z 23 Acked 1022 Write 1 Done Sent Done false
232452023-09-22T23:34:51.905Z 24 Acked 1023 Write 7 Done Sent Done false
232462023-09-22T23:34:51.908Z 25 Acked 1024 Read 9 Done Sent Done false
232472023-09-22T23:34:51.911Z 26 Acked 1025 Write 6 Done Sent Done false
232482023-09-22T23:34:51.914Z 27 Acked 1026 Write 6 Done Sent Done false
232492023-09-22T23:34:51.916Z 28 Acked 1027 Read 6 Done Sent Done false
232502023-09-22T23:34:51.919Z 29 Acked 1028 Read 5 Done Sent Done false
232512023-09-22T23:34:51.922Z 30 Acked 1029 Read 7 Sent Sent Done false
232522023-09-22T23:34:51.925Z STATES DS:0 DS:1 DS:2 TOTAL
232532023-09-22T23:34:51.929Z New 0 0 0 0
232542023-09-22T23:34:51.932Z Sent 1 14 0 15
232552023-09-22T23:34:51.935Z Done 13 0 14 27
232562023-09-22T23:34:51.938Z Skipped 0 0 0 0
232572023-09-22T23:34:51.941Z Error 0 0 0 0
232582023-09-22T23:34:51.944ZLast Flush: 1015 1015 1015
232592023-09-22T23:34:51.947ZDownstairs last five completed: 1015 1014 1013 1012 1011 1010
232602023-09-22T23:34:51.949ZUpstairs last five completed: 30 29 28 27 26 25
232612023-09-22T23:34:51.952ZWrote out file "/tmp/test_repair_verify.data"
232622023-09-22T23:34:51.955Z
232632023-09-22T23:34:51.958ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
232642023-09-22T23:34:51.961Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.529789979Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1888}
232652023-09-22T23:34:51.964Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.530083384Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1888}
232662023-09-22T23:34:51.967Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.530106261Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1888}
232672023-09-22T23:34:51.970Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.559602054Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1888}
232682023-09-22T23:34:51.973Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.559636894Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1888}
232692023-09-22T23:34:51.976Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.559656023Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1888}
232702023-09-22T23:34:51.979Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.590869923Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1888}
232712023-09-22T23:34:51.982Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.590901951Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1888}
232722023-09-22T23:34:51.985Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:51.590920995Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1888}
232732023-09-22T23:34:51.988ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
232742023-09-22T23:34:51.991Z 0 000-019 176 176 176 285 285 285 F F F
232752023-09-22T23:34:51.994Z 1 020-039 176 176 176 285 285 285 F F F
232762023-09-22T23:34:51.997Z 2 040-059 176 176 176 285 285 285 F F F
232772023-09-22T23:34:52.001Z 3 060-079 176 176 176 286 286 286 T F T <---
232782023-09-22T23:34:52.004Z 4 080-099 174 174 174 281 281 281 F F F
232792023-09-22T23:34:52.007Z 5 100-119 163 163 163 260 260 260 F F F
232802023-09-22T23:34:52.010Z 6 120-139 168 168 168 270 270 270 F F F
232812023-09-22T23:34:52.012Z 7 140-159 166 166 166 267 267 267 T F T <---
232822023-09-22T23:34:52.015Z 8 160-179 174 174 174 281 281 281 F F F
232832023-09-22T23:34:52.018Z 9 180-199 176 176 176 285 285 285 F F F
232842023-09-22T23:34:52.021Z 10 200-219 172 172 172 278 278 278 F F F
232852023-09-22T23:34:52.024Z 11 220-239 170 170 170 275 275 275 F F F
232862023-09-22T23:34:52.027Z 12 240-259 174 174 174 280 280 280 F F F
232872023-09-22T23:34:52.030Z 13 260-279 172 172 172 278 278 278 F F F
232882023-09-22T23:34:52.033Z 14 280-299 174 174 174 281 281 281 F F F
232892023-09-22T23:34:52.036Z 15 300-319 174 174 174 282 282 282 F F F
232902023-09-22T23:34:52.038Z 16 320-339 176 176 176 285 285 285 F F F
232912023-09-22T23:34:52.041Z 17 340-359 175 175 175 283 283 283 T F T <---
232922023-09-22T23:34:52.044Z 18 360-379 171 171 171 276 276 276 F F F
232932023-09-22T23:34:52.046Z 19 380-399 173 173 173 279 279 279 F F F
232942023-09-22T23:34:52.049Z 20 400-419 169 169 169 273 273 273 F F F
232952023-09-22T23:34:52.053Z 21 420-439 171 171 171 276 276 276 F F F
232962023-09-22T23:34:52.056Z 22 440-459 175 175 175 283 283 283 F F F
232972023-09-22T23:34:52.059Z 23 460-479 172 172 172 278 278 278 T F T <---
232982023-09-22T23:34:52.062Z 24 480-499 176 176 176 284 284 284 T F T <---
232992023-09-22T23:34:52.065Z 25 500-519 174 174 174 281 281 281 F F F
233002023-09-22T23:34:52.068Z 26 520-539 174 174 174 282 282 282 T F T <---
233012023-09-22T23:34:52.071Z 27 540-559 169 169 169 273 273 273 F F F
233022023-09-22T23:34:52.074Z 28 560-579 168 168 168 270 270 270 F F F
233032023-09-22T23:34:52.077Z 29 580-599 166 166 166 265 265 265 T F T <---
233042023-09-22T23:34:52.080ZMax gen: 176, Max flush: 286
233052023-09-22T23:34:52.083ZA Difference in extent metadata is expected here
233062023-09-22T23:34:52.086ZOn loop 88
233072023-09-22T23:34:52.089Z
233082023-09-22T23:34:52.092ZVerifying data now
233092023-09-22T23:34:57.542ZNo change after verify
233102023-09-22T23:34:57.546ZLoop: 88 Downstairs dump after verify (and repair):
233112023-09-22T23:34:57.561Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.558844542Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1893}
233122023-09-22T23:34:57.563Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.559245364Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1893}
233132023-09-22T23:34:57.566Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.559271011Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1893}
233142023-09-22T23:34:57.579Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.577563572Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1893}
233152023-09-22T23:34:57.582Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.577617915Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1893}
233162023-09-22T23:34:57.585Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.577638479Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1893}
233172023-09-22T23:34:57.590Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.588470939Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1893}
233182023-09-22T23:34:57.593Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.588524373Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1893}
233192023-09-22T23:34:57.595Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.588544784Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1893}
233202023-09-22T23:34:57.602ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
233212023-09-22T23:34:57.606Z 0 000-019 176 176 176 285 285 285 F F F
233222023-09-22T23:34:57.610Z 1 020-039 176 176 176 285 285 285 F F F
233232023-09-22T23:34:57.612Z 2 040-059 176 176 176 285 285 285 F F F
233242023-09-22T23:34:57.615Z 3 060-079 177 177 177 287 287 287 F F F
233252023-09-22T23:34:57.618Z 4 080-099 174 174 174 281 281 281 F F F
233262023-09-22T23:34:57.621Z 5 100-119 163 163 163 260 260 260 F F F
233272023-09-22T23:34:57.625Z 6 120-139 168 168 168 270 270 270 F F F
233282023-09-22T23:34:57.627Z 7 140-159 177 177 177 287 287 287 F F F
233292023-09-22T23:34:57.630Z 8 160-179 174 174 174 281 281 281 F F F
233302023-09-22T23:34:57.635Z 9 180-199 176 176 176 285 285 285 F F F
233312023-09-22T23:34:57.641Z 10 200-219 172 172 172 278 278 278 F F F
233322023-09-22T23:34:57.644Z 11 220-239 170 170 170 275 275 275 F F F
233332023-09-22T23:34:57.647Z 12 240-259 174 174 174 280 280 280 F F F
233342023-09-22T23:34:57.650Z 13 260-279 172 172 172 278 278 278 F F F
233352023-09-22T23:34:57.653Z 14 280-299 174 174 174 281 281 281 F F F
233362023-09-22T23:34:57.656Z 15 300-319 174 174 174 282 282 282 F F F
233372023-09-22T23:34:57.659Z 16 320-339 176 176 176 285 285 285 F F F
233382023-09-22T23:34:57.664Z 17 340-359 177 177 177 287 287 287 F F F
233392023-09-22T23:34:57.667Z 18 360-379 171 171 171 276 276 276 F F F
233402023-09-22T23:34:57.670Z 19 380-399 173 173 173 279 279 279 F F F
233412023-09-22T23:34:57.673Z 20 400-419 169 169 169 273 273 273 F F F
233422023-09-22T23:34:57.675Z 21 420-439 171 171 171 276 276 276 F F F
233432023-09-22T23:34:57.678Z 22 440-459 175 175 175 283 283 283 F F F
233442023-09-22T23:34:57.681Z 23 460-479 177 177 177 287 287 287 F F F
233452023-09-22T23:34:57.684Z 24 480-499 177 177 177 287 287 287 F F F
233462023-09-22T23:34:57.687Z 25 500-519 174 174 174 281 281 281 F F F
233472023-09-22T23:34:57.690Z 26 520-539 177 177 177 287 287 287 F F F
233482023-09-22T23:34:57.693Z 27 540-559 169 169 169 273 273 273 F F F
233492023-09-22T23:34:57.696Z 28 560-579 168 168 168 270 270 270 F F F
233502023-09-22T23:34:57.699Z 29 580-599 177 177 177 287 287 287 F F F
233512023-09-22T23:34:57.713ZMax gen: 177, Max flush: 287
233522023-09-22T23:34:57.716Z
233532023-09-22T23:34:57.722ZBegin loop 89
233542023-09-22T23:34:57.727ZDownstairs to restart: 1
233552023-09-22T23:34:57.730Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.638185588Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233562023-09-22T23:34:57.734Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.638583719Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233572023-09-22T23:34:57.740Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.638636839Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233582023-09-22T23:34:57.743Z{"msg":"Crucible stats registered with UUID: 97ce1a50-a89c-41f0-9fad-33e405e18c65","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.638697585Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233592023-09-22T23:34:57.746Z{"msg":"Crucible 97ce1a50-a89c-41f0-9fad-33e405e18c65 has session id: 3f006bf6-0bc9-4504-aeea-c15a8f35cb44","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.638779636Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233602023-09-22T23:34:57.749ZCrucible runtime is spawned
233612023-09-22T23:34:57.752Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.63914586Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895,"task":"up_listen"}
233622023-09-22T23:34:57.759Z{"msg":"Wait for all three downstairs to come online","v":0The guest has requested activation with gen:178
233632023-09-22T23:34:57.763Z,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.639324431Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233642023-09-22T23:34:57.765Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.639391181Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233652023-09-22T23:34:57.769Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.639458969Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895,"looper":"1"}
233662023-09-22T23:34:57.772Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.639601012Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895,"looper":"2"}
233672023-09-22T23:34:57.777Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.639817871Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895,"looper":"0"}
233682023-09-22T23:34:57.781Z{"msg":"[2] 97ce1a50-a89c-41f0-9fad-33e405e18c65 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.640073742Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895,"looper":"2"}
233692023-09-22T23:34:57.784Z{"msg":"[0] 97ce1a50-a89c-41f0-9fad-33e405e18c65 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.640220604Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895,"looper":"0"}
233702023-09-22T23:34:57.787Z{"msg":"97ce1a50-a89c-41f0-9fad-33e405e18c65 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.640332108Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233712023-09-22T23:34:57.790Z{"msg":"Set desired generation to :178","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.640369388Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233722023-09-22T23:34:57.792Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.640573796Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233732023-09-22T23:34:57.796Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.640684679Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233742023-09-22T23:34:57.798Z{"msg":"[2] 97ce1a50-a89c-41f0-9fad-33e405e18c65 (eb55f23e-efd7-4a64-9b1c-298901dabbce) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.640980792Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233752023-09-22T23:34:57.801Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.641045366Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233762023-09-22T23:34:57.805Z{"msg":"[2] client is_active_req TRUE, promote! session eb55f23e-efd7-4a64-9b1c-298901dabbce","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.641089495Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233772023-09-22T23:34:57.808Z{"msg":"[0] 97ce1a50-a89c-41f0-9fad-33e405e18c65 (eb55f23e-efd7-4a64-9b1c-298901dabbce) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.641113866Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233782023-09-22T23:34:57.811Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.641151531Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233792023-09-22T23:34:57.814Z{"msg":"[0] client is_active_req TRUE, promote! session eb55f23e-efd7-4a64-9b1c-298901dabbce","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.641174283Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233802023-09-22T23:34:57.817Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.641870401Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233812023-09-22T23:34:57.820Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.64193028Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233822023-09-22T23:34:57.823Z{"msg":"97ce1a50-a89c-41f0-9fad-33e405e18c65 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.642106385Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233832023-09-22T23:34:57.828Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.642244265Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233842023-09-22T23:34:57.831Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.642307772Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233852023-09-22T23:34:57.834Z{"msg":"97ce1a50-a89c-41f0-9fad-33e405e18c65 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.642349289Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233862023-09-22T23:34:57.837Z{"msg":"[2] 97ce1a50-a89c-41f0-9fad-33e405e18c65 (eb55f23e-efd7-4a64-9b1c-298901dabbce) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.644110988Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233872023-09-22T23:34:57.840Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.644194715Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233882023-09-22T23:34:57.843Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:34:57.64426881Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233892023-09-22T23:34:57.846Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.644346728Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233902023-09-22T23:34:57.849Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.644383207Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233912023-09-22T23:34:57.852Z{"msg":"97ce1a50-a89c-41f0-9fad-33e405e18c65 WaitActive New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.644418769Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233922023-09-22T23:34:57.855Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.644485744Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233932023-09-22T23:34:57.857Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.644549944Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233942023-09-22T23:34:57.861Z{"msg":"[0] 97ce1a50-a89c-41f0-9fad-33e405e18c65 (eb55f23e-efd7-4a64-9b1c-298901dabbce) WaitActive New WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.644622943Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233952023-09-22T23:34:57.864Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.644694723Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233962023-09-22T23:34:57.867Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:34:57.644763003Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233972023-09-22T23:34:57.870Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.644832981Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233982023-09-22T23:34:57.873Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.644905205Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
233992023-09-22T23:34:57.896Z{"msg":"97ce1a50-a89c-41f0-9fad-33e405e18c65 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.644947484Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234002023-09-22T23:34:57.899Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.644985903Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234012023-09-22T23:34:57.902Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:57.645023459Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234022023-09-22T23:34:58.644Z{"msg":"[1] 97ce1a50-a89c-41f0-9fad-33e405e18c65 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.641710697Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895,"looper":"1"}
234032023-09-22T23:34:58.647Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.641797885Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234042023-09-22T23:34:58.650Z{"msg":"[1] 97ce1a50-a89c-41f0-9fad-33e405e18c65 (eb55f23e-efd7-4a64-9b1c-298901dabbce) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.642667068Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234052023-09-22T23:34:58.653Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.642722166Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234062023-09-22T23:34:58.656Z{"msg":"[1] client is_active_req TRUE, promote! session eb55f23e-efd7-4a64-9b1c-298901dabbce","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.642767183Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234072023-09-22T23:34:58.659Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.643341916Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234082023-09-22T23:34:58.662Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.643397975Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234092023-09-22T23:34:58.665Z{"msg":"97ce1a50-a89c-41f0-9fad-33e405e18c65 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.643441803Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234102023-09-22T23:34:58.668Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.645751627Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234112023-09-22T23:34:58.671Z{"msg":"[1] 97ce1a50-a89c-41f0-9fad-33e405e18c65 (eb55f23e-efd7-4a64-9b1c-298901dabbce) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.64590225Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234122023-09-22T23:34:58.674Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.645980636Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234132023-09-22T23:34:58.678Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:34:58.646004856Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234142023-09-22T23:34:58.681Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.646043254Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234152023-09-22T23:34:58.684Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.646082364Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234162023-09-22T23:34:58.687Z{"msg":"97ce1a50-a89c-41f0-9fad-33e405e18c65 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.646122687Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234172023-09-22T23:34:58.690Z{"msg":"[0]R flush_numbers[0..12]: [285, 285, 285, 287, 281, 260, 270, 287, 281, 285, 278, 275]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.646163069Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234182023-09-22T23:34:58.694Z{"msg":"[0]R generation[0..12]: [176, 176, 176, 177, 174, 163, 168, 177, 174, 176, 172, 170]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.646203457Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234192023-09-22T23:34:58.697Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.646227061Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234202023-09-22T23:34:58.700Z{"msg":"[1]R flush_numbers[0..12]: [285, 285, 285, 287, 281, 260, 270, 287, 281, 285, 278, 275]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.646286675Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234212023-09-22T23:34:58.703ZThe guest has finished waiting for activation with:178
234222023-09-22T23:34:58.707Z{"msg":"Wait for a query_work_queue command to finish before sending IO
234232023-09-22T23:34:58.710Z[1]R generation[0..12]: [176, 176, 176, 177, 174, 163, 168, 177, 174, 176, 172, 170]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.646360821Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234242023-09-22T23:34:58.714Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.646427472Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234252023-09-22T23:34:58.717Z{"msg":"[2]R flush_numbers[0..12]: [285, 285, 285, 287, 281, 260, 270, 287, 281, 285, 278, 275]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.646498849Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234262023-09-22T23:34:58.720Z{"msg":"[2]R generation[0..12]: [176, 176, 176, 177, 174, 163, 168, 177, 174, 176, 172, 170]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.64656921Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234272023-09-22T23:34:58.732Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.646618136Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234282023-09-22T23:34:58.735Z{"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.646681904Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234292023-09-22T23:34:58.738Z{"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.646739981Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234302023-09-22T23:34:58.741Z{"msg":"Next flush: 288","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.646795827Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234312023-09-22T23:34:58.744Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.646865505Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234322023-09-22T23:34:58.747Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.646926179Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234332023-09-22T23:34:58.750Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.646994536Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234342023-09-22T23:34:58.753Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.647049847Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234352023-09-22T23:34:58.761Z{"msg":"97ce1a50-a89c-41f0-9fad-33e405e18c65 is now active with session: eb55f23e-efd7-4a64-9b1c-298901dabbce","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.647110523Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234362023-09-22T23:34:58.764Z{"msg":"97ce1a50-a89c-41f0-9fad-33e405e18c65 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.647172055Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234372023-09-22T23:34:58.767Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.647239905Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234382023-09-22T23:34:58.769Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.647286008Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234392023-09-22T23:34:58.773Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.647329208Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234402023-09-22T23:34:58.776Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.647393802Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234412023-09-22T23:34:58.779Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.647454207Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234422023-09-22T23:34:58.782Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.647497651Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234432023-09-22T23:34:58.785Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
234442023-09-22T23:34:58.788Z,"level":30,"time":"2023-09-22T23:34:58.647621353Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234452023-09-22T23:34:58.791Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.647694413Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234462023-09-22T23:34:58.794Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.647762141Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234472023-09-22T23:34:58.797Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.64782514Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234482023-09-22T23:34:58.801Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.647936823Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1895}
234492023-09-22T23:34:58.804ZLoading write count information from file "/tmp/test_repair_verify.data"
234502023-09-22T23:34:58.808ZRun Repair workload
234512023-09-22T23:34:58.810Z01/30 Write block 353 len 28672 data: 16 15 15 14 12 16 19
234522023-09-22T23:34:58.814Z02/30 Read block 150 len 40960
234532023-09-22T23:34:58.817Z03/30 Write block 307 len 32768 data: 15 13 12 9 8 8 10 13
234542023-09-22T23:34:58.821Z04/30 Read block 166 len 32768
234552023-09-22T23:34:58.824Z05/30 Flush
234562023-09-22T23:34:58.827Z06/30 Write block 557 len 4096 data: 9
234572023-09-22T23:34:58.831Z07/30 Read block 113 len 36864
234582023-09-22T23:34:58.836Z08/30 Read block 91 len 8192
234592023-09-22T23:34:58.840Z09/30 Read block 438 len 20480
234602023-09-22T23:34:58.843Z10/30 Read block 429 len 8192
234612023-09-22T23:34:58.851Z11/30 Read block 527 len 4096
234622023-09-22T23:34:58.855Z12/30 Write block 60 len 4096 data: 17
234632023-09-22T23:34:58.858Z13/30 Read block 489 len 28672
234642023-09-22T23:34:58.861Z14/30 Read block 441 len 4096
234652023-09-22T23:34:58.864Z15/30 Read block 121 len 36864
234662023-09-22T23:34:58.867Z16/30 Write block 571 len 24576 data: 9 6 6 6 9 10
234672023-09-22T23:34:58.871Z17/30 Write block 118 len 24576 data: 11 11 12 12 13 14
234682023-09-22T23:34:58.874Z18/30 Write block 431 len 32768 data: 15 17 17 15 18 17 16 18
234692023-09-22T23:34:58.877Z19/30 Write block 44 len 36864 data: 12 13 12 14 12 13 11 13 10
234702023-09-22T23:34:58.880Z20/30 Write block 21 len 28672 data: 6 9 7 10 11 12 14
234712023-09-22T23:34:58.883Z21/30 Flush
234722023-09-22T23:34:58.886Z22/30 Write block 367 len 24576 data: 10 13 12 9 10 10
234732023-09-22T23:34:58.889Z23/30 Read block 266 len 24576
234742023-09-22T23:34:58.892Z24/30 Write block 304 len 4096 data: 14
234752023-09-22T23:34:58.895Z25/30 Write block 201 len 16384 data: 12 16 19 18
234762023-09-22T23:34:58.917Z26/30 Write block 28 len 8192 data: 11 9
234772023-09-22T23:34:58.924Z27/30 Write block 573 len 24576 data: 7 7 10 11 12 12
234782023-09-22T23:34:58.929Z28/30 Read block 460 len 4096
234792023-09-22T23:34:58.932Z29/30 Read block 439 len 20480
234802023-09-22T23:34:58.936Z30/30 Write block 305 len 28672 data: 14 15 16 14 13 10 9
234812023-09-22T23:34:58.940Z----------------------------------------------------------------
234822023-09-22T23:34:58.943Z Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30
234832023-09-22T23:34:58.946ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
234842023-09-22T23:34:58.949Z 1 Acked 1000 Write 7 Done Sent Done false
234852023-09-22T23:34:58.952Z 2 Acked 1001 Read 10 Done Sent Done false
234862023-09-22T23:34:58.955Z 3 Acked 1002 Write 8 Done Sent Done false
234872023-09-22T23:34:58.958Z 4 Acked 1003 Read 8 Done Sent Done false
234882023-09-22T23:34:58.962Z 5 Acked 1004 Flush 0 Done Sent Done false
234892023-09-22T23:34:58.964Z 6 Acked 1005 Write 1 Done Sent Done false
234902023-09-22T23:34:58.967Z 7 Acked 1006 Read 9 Done Sent Done false
234912023-09-22T23:34:58.970Z 8 Acked 1007 Read 2 Done Sent Done false
234922023-09-22T23:34:58.973Z 9 Acked 1008 Read 5 Done Sent Done false
234932023-09-22T23:34:58.976Z 10 Acked 1009 Read 2 Done Sent Done false
234942023-09-22T23:34:58.978Z 11 Acked 1010 Read 1 Done Sent Done false
234952023-09-22T23:34:58.985Z 12 Acked 1011 Write 1 Done Sent Done false
234962023-09-22T23:34:58.988Z 13 Acked 1012 Read 7 Done Sent Done false
234972023-09-22T23:34:58.990Z 14 Acked 1013 Read 1 Done Sent Done false
234982023-09-22T23:34:58.993Z 15 Acked 1014 Read 9 Done Sent Done false
234992023-09-22T23:34:58.996Z 16 Acked 1015 Write 6 Done Sent Done false
235002023-09-22T23:34:59.002Z 17 Acked 1016 Write 6 Done Sent Done false
235012023-09-22T23:34:59.005Z 18 Acked 1017 Write 8 Done Sent Done false
235022023-09-22T23:34:59.008Z 19 Acked 1018 Write 9 Done Sent Done false
235032023-09-22T23:34:59.011Z 20 Acked 1019 Write 7 Done Sent Done false
235042023-09-22T23:34:59.013Z 21 Acked 1020 Flush 0 Done Sent Done false
235052023-09-22T23:34:59.016Z 22 Acked 1021 Write 6 Done Sent Done false
235062023-09-22T23:34:59.019Z 23 Acked 1022 Read 6 Done Sent Done false
235072023-09-22T23:34:59.022Z 24 Acked 1023 Write 1 Done Sent Done false
235082023-09-22T23:34:59.025Z 25 Acked 1024 Write 4 Done Sent Done false
235092023-09-22T23:34:59.029Z 26 Acked 1025 Write 2 Done Sent Done false
235102023-09-22T23:34:59.032Z 27 Acked 1026 Write 6 Done Sent Done false
235112023-09-22T23:34:59.035Z 28 Acked 1027 Read 1 Done Sent Done false
235122023-09-22T23:34:59.038Z 29 Acked 1028 Read 5 Done Sent Done false
235132023-09-22T23:34:59.041Z 30 Acked 1029 Write 7 Sent Sent Sent false
235142023-09-22T23:34:59.044Z STATES DS:0 DS:1 DS:2 TOTAL
235152023-09-22T23:34:59.047Z New 0 0 0 0
235162023-09-22T23:34:59.050Z Sent 1 30 1 32
235172023-09-22T23:34:59.054Z Done 29 0 29 58
235182023-09-22T23:34:59.058Z Skipped 0 0 0 0
235192023-09-22T23:34:59.061Z Error 0 0 0 0
235202023-09-22T23:34:59.064ZLast Flush: 1020 0 1020
235212023-09-22T23:34:59.067ZDownstairs last five completed:
235222023-09-22T23:34:59.070ZUpstairs last five completed: 30 29 28 27 26 25
235232023-09-22T23:34:59.072ZWrote out file "/tmp/test_repair_verify.data"
235242023-09-22T23:34:59.075Z
235252023-09-22T23:34:59.079ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
235262023-09-22T23:34:59.082ZError: Difference in extent metadata found!
235272023-09-22T23:34:59.085Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.780079277Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1896}
235282023-09-22T23:34:59.088Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.780379484Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1896}
235292023-09-22T23:34:59.091Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.780403158Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1896}
235302023-09-22T23:34:59.094Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.807668897Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1896}
235312023-09-22T23:34:59.097Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.807707383Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1896}
235322023-09-22T23:34:59.099Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.807726754Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1896}
235332023-09-22T23:34:59.103Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.822977197Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1896}
235342023-09-22T23:34:59.106Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.823008513Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1896}
235352023-09-22T23:34:59.109Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:34:58.823027252Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1896}
235362023-09-22T23:34:59.113ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
235372023-09-22T23:34:59.116Z 0 000-019 176 176 176 285 285 285 F F F
235382023-09-22T23:34:59.119Z 1 020-039 178 176 178 289 285 289 T F T <---
235392023-09-22T23:34:59.122Z 2 040-059 178 176 178 289 285 289 F F F <---
235402023-09-22T23:34:59.126Z 3 060-079 178 177 178 289 287 289 F F F <---
235412023-09-22T23:34:59.130Z 4 080-099 174 174 174 281 281 281 F F F
235422023-09-22T23:34:59.133Z 5 100-119 178 163 178 289 260 289 F F F <---
235432023-09-22T23:34:59.136Z 6 120-139 178 168 178 289 270 289 F F F <---
235442023-09-22T23:34:59.139Z 7 140-159 177 177 177 287 287 287 F F F
235452023-09-22T23:34:59.142Z 8 160-179 174 174 174 281 281 281 F F F
235462023-09-22T23:34:59.145Z 9 180-199 176 176 176 285 285 285 F F F
235472023-09-22T23:34:59.148Z 10 200-219 172 172 172 278 278 278 T F T <---
235482023-09-22T23:34:59.151Z 11 220-239 170 170 170 275 275 275 F F F
235492023-09-22T23:34:59.155Z 12 240-259 174 174 174 280 280 280 F F F
235502023-09-22T23:34:59.158Z 13 260-279 172 172 172 278 278 278 F F F
235512023-09-22T23:34:59.161Z 14 280-299 174 174 174 281 281 281 F F F
235522023-09-22T23:34:59.164Z 15 300-319 178 174 178 288 282 288 T F T <---
235532023-09-22T23:34:59.167Z 16 320-339 176 176 176 285 285 285 F F F
235542023-09-22T23:34:59.170Z 17 340-359 178 177 178 288 287 288 F F F <---
235552023-09-22T23:34:59.173Z 18 360-379 171 171 171 276 276 276 T F T <---
235562023-09-22T23:34:59.176Z 19 380-399 173 173 173 279 279 279 F F F
235572023-09-22T23:34:59.179Z 20 400-419 169 169 169 273 273 273 F F F
235582023-09-22T23:34:59.182Z 21 420-439 178 171 178 289 276 289 F F F <---
235592023-09-22T23:34:59.186Z 22 440-459 175 175 175 283 283 283 F F F
235602023-09-22T23:34:59.189Z 23 460-479 177 177 177 287 287 287 F F F
235612023-09-22T23:34:59.193Z 24 480-499 177 177 177 287 287 287 F F F
235622023-09-22T23:34:59.195Z 25 500-519 174 174 174 281 281 281 F F F
235632023-09-22T23:34:59.199Z 26 520-539 177 177 177 287 287 287 F F F
235642023-09-22T23:34:59.202Z 27 540-559 178 169 178 289 273 289 F F F <---
235652023-09-22T23:34:59.205Z 28 560-579 178 168 178 289 270 289 T F T <---
235662023-09-22T23:34:59.208Z 29 580-599 177 177 177 287 287 287 F F F
235672023-09-22T23:34:59.211ZMax gen: 178, Max flush: 289
235682023-09-22T23:34:59.220ZA Difference in extent metadata is expected here
235692023-09-22T23:34:59.223ZOn loop 89
235702023-09-22T23:34:59.226Z
235712023-09-22T23:34:59.228ZVerifying data now
235722023-09-22T23:35:05.178ZNo change after verify
235732023-09-22T23:35:05.182ZLoop: 89 Downstairs dump after verify (and repair):
235742023-09-22T23:35:05.197Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.195185349Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1901}
235752023-09-22T23:35:05.200Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.195586599Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1901}
235762023-09-22T23:35:05.203Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.195611544Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1901}
235772023-09-22T23:35:05.216Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.214441217Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1901}
235782023-09-22T23:35:05.219Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.21449433Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1901}
235792023-09-22T23:35:05.222Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.214515454Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1901}
235802023-09-22T23:35:05.227Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.225705901Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1901}
235812023-09-22T23:35:05.231Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.22575774Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1901}
235822023-09-22T23:35:05.234Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.225778119Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1901}
235832023-09-22T23:35:05.238ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
235842023-09-22T23:35:05.241Z 0 000-019 176 176 176 285 285 285 F F F
235852023-09-22T23:35:05.244Z 1 020-039 179 179 179 290 290 290 F F F
235862023-09-22T23:35:05.247Z 2 040-059 178 178 178 289 289 289 F F F
235872023-09-22T23:35:05.250Z 3 060-079 178 178 178 289 289 289 F F F
235882023-09-22T23:35:05.253Z 4 080-099 174 174 174 281 281 281 F F F
235892023-09-22T23:35:05.256Z 5 100-119 178 178 178 289 289 289 F F F
235902023-09-22T23:35:05.259Z 6 120-139 178 178 178 289 289 289 F F F
235912023-09-22T23:35:05.262Z 7 140-159 177 177 177 287 287 287 F F F
235922023-09-22T23:35:05.265Z 8 160-179 174 174 174 281 281 281 F F F
235932023-09-22T23:35:05.268Z 9 180-199 176 176 176 285 285 285 F F F
235942023-09-22T23:35:05.271Z 10 200-219 179 179 179 290 290 290 F F F
235952023-09-22T23:35:05.274Z 11 220-239 170 170 170 275 275 275 F F F
235962023-09-22T23:35:05.277Z 12 240-259 174 174 174 280 280 280 F F F
235972023-09-22T23:35:05.280Z 13 260-279 172 172 172 278 278 278 F F F
235982023-09-22T23:35:05.284Z 14 280-299 174 174 174 281 281 281 F F F
235992023-09-22T23:35:05.288Z 15 300-319 179 179 179 290 290 290 F F F
236002023-09-22T23:35:05.291Z 16 320-339 176 176 176 285 285 285 F F F
236012023-09-22T23:35:05.294Z 17 340-359 178 178 178 288 288 288 F F F
236022023-09-22T23:35:05.298Z 18 360-379 179 179 179 290 290 290 F F F
236032023-09-22T23:35:05.302Z 19 380-399 173 173 173 279 279 279 F F F
236042023-09-22T23:35:05.305Z 20 400-419 169 169 169 273 273 273 F F F
236052023-09-22T23:35:05.308Z 21 420-439 178 178 178 289 289 289 F F F
236062023-09-22T23:35:05.311Z 22 440-459 175 175 175 283 283 283 F F F
236072023-09-22T23:35:05.314Z 23 460-479 177 177 177 287 287 287 F F F
236082023-09-22T23:35:05.317Z 24 480-499 177 177 177 287 287 287 F F F
236092023-09-22T23:35:05.320Z 25 500-519 174 174 174 281 281 281 F F F
236102023-09-22T23:35:05.324Z 26 520-539 177 177 177 287 287 287 F F F
236112023-09-22T23:35:05.327Z 27 540-559 178 178 178 289 289 289 F F F
236122023-09-22T23:35:05.330Z 28 560-579 179 179 179 290 290 290 F F F
236132023-09-22T23:35:05.333Z 29 580-599 177 177 177 287 287 287 F F F
236142023-09-22T23:35:05.336ZMax gen: 179, Max flush: 290
236152023-09-22T23:35:05.339Z
236162023-09-22T23:35:05.342ZBegin loop 90
236172023-09-22T23:35:05.344ZDownstairs to restart: 1
236182023-09-22T23:35:05.347Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.274604863Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236192023-09-22T23:35:05.350Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.274976846Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236202023-09-22T23:35:05.353Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.275033341Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236212023-09-22T23:35:05.356Z{"msg":"Crucible stats registered with UUID: c9cbfa6b-b641-4753-932e-e5ca1b09c279","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.275057281Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236222023-09-22T23:35:05.359Z{"msg":"Crucible c9cbfa6b-b641-4753-932e-e5ca1b09c279 has session id: 5f6d01e3-1f3d-4074-a278-e0ed9508a154","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.27515483Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236232023-09-22T23:35:05.362ZCrucible runtime is spawned
236242023-09-22T23:35:05.365ZThe guest has requested activation with gen:180
236252023-09-22T23:35:05.368Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.275484439Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903,"task":"up_listen"}
236262023-09-22T23:35:05.371Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.275592666Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236272023-09-22T23:35:05.374Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.275639839Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236282023-09-22T23:35:05.382Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.27566261Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903,"looper":"2"}
236292023-09-22T23:35:05.386Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.275893136Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903,"looper":"1"}
236302023-09-22T23:35:05.390Z{"msg":"[2] c9cbfa6b-b641-4753-932e-e5ca1b09c279 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.275972034Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903,"looper":"2"}
236312023-09-22T23:35:05.393Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.276011777Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903,"looper":"0"}
236322023-09-22T23:35:05.396Z{"msg":"[0] c9cbfa6b-b641-4753-932e-e5ca1b09c279 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.276244134Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903,"looper":"0"}
236332023-09-22T23:35:05.399Z{"msg":"c9cbfa6b-b641-4753-932e-e5ca1b09c279 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.276379616Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236342023-09-22T23:35:05.403Z{"msg":"Set desired generation to :180","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.276435147Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236352023-09-22T23:35:05.406Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.276496161Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236362023-09-22T23:35:05.409Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.276743895Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236372023-09-22T23:35:05.412Z{"msg":"[2] c9cbfa6b-b641-4753-932e-e5ca1b09c279 (c3fdb5c1-897c-4d6e-b241-8e01c7ed852e) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.277134806Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236382023-09-22T23:35:05.415Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.277190449Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236392023-09-22T23:35:05.424Z{"msg":"[2] client is_active_req TRUE, promote! session c3fdb5c1-897c-4d6e-b241-8e01c7ed852e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.277236503Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236402023-09-22T23:35:05.432Z{"msg":"[0] c9cbfa6b-b641-4753-932e-e5ca1b09c279 (c3fdb5c1-897c-4d6e-b241-8e01c7ed852e) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.277439492Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236412023-09-22T23:35:05.435Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.277496169Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236422023-09-22T23:35:05.438Z{"msg":"[0] client is_active_req TRUE, promote! session c3fdb5c1-897c-4d6e-b241-8e01c7ed852e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.277522096Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236432023-09-22T23:35:05.441Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.277937594Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236442023-09-22T23:35:05.444Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.277997117Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236452023-09-22T23:35:05.447Z{"msg":"c9cbfa6b-b641-4753-932e-e5ca1b09c279 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.278060984Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236462023-09-22T23:35:05.450Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.278216165Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236472023-09-22T23:35:05.453Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.278277835Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236482023-09-22T23:35:05.456Z{"msg":"c9cbfa6b-b641-4753-932e-e5ca1b09c279 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.278319353Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236492023-09-22T23:35:05.458Z{"msg":"[2] c9cbfa6b-b641-4753-932e-e5ca1b09c279 (c3fdb5c1-897c-4d6e-b241-8e01c7ed852e) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.280809588Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236502023-09-22T23:35:05.461Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.280909289Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236512023-09-22T23:35:05.468Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:35:05.280985359Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236522023-09-22T23:35:05.470Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.281028156Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236532023-09-22T23:35:05.473Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.281096523Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236542023-09-22T23:35:05.476Z{"msg":"c9cbfa6b-b641-4753-932e-e5ca1b09c279 WaitActive New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.281170932Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236552023-09-22T23:35:05.479Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.281241662Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236562023-09-22T23:35:05.482Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.281313823Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236572023-09-22T23:35:05.485Z{"msg":"[0] c9cbfa6b-b641-4753-932e-e5ca1b09c279 (c3fdb5c1-897c-4d6e-b241-8e01c7ed852e) WaitActive New WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.281388869Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236582023-09-22T23:35:05.488Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.281437647Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236592023-09-22T23:35:05.491Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:35:05.28147733Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236602023-09-22T23:35:05.499Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.281516155Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236612023-09-22T23:35:05.502Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.281554608Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236622023-09-22T23:35:05.505Z{"msg":"c9cbfa6b-b641-4753-932e-e5ca1b09c279 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.281593546Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236632023-09-22T23:35:05.507Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.281632167Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236642023-09-22T23:35:05.510Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:05.281651698Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236652023-09-22T23:35:06.279Z{"msg":"[1] c9cbfa6b-b641-4753-932e-e5ca1b09c279 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.27727826Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903,"looper":"1"}
236662023-09-22T23:35:06.283Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.277363106Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236672023-09-22T23:35:06.286Z{"msg":"[1] c9cbfa6b-b641-4753-932e-e5ca1b09c279 (c3fdb5c1-897c-4d6e-b241-8e01c7ed852e) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.278543056Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236682023-09-22T23:35:06.291Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.278622793Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236692023-09-22T23:35:06.296Z{"msg":"[1] client is_active_req TRUE, promote! session c3fdb5c1-897c-4d6e-b241-8e01c7ed852e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.278696448Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236702023-09-22T23:35:06.300Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.279273262Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236712023-09-22T23:35:06.303Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.279339235Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236722023-09-22T23:35:06.310Z{"msg":"c9cbfa6b-b641-4753-932e-e5ca1b09c279 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.279384073Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236732023-09-22T23:35:06.314Z{"msg":"[1] c9cbfa6b-b641-4753-932e-e5ca1b09c279 (c3fdb5c1-897c-4d6e-b241-8e01c7ed852e) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.281777131Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236742023-09-22T23:35:06.316Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.281870289Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236752023-09-22T23:35:06.319Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:35:06.281894729Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236762023-09-22T23:35:06.324Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.281918804Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236772023-09-22T23:35:06.327Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.281962494Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236782023-09-22T23:35:06.330Z{"msg":"c9cbfa6b-b641-4753-932e-e5ca1b09c279 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.282003398Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236792023-09-22T23:35:06.334Z{"msg":"[0]R flush_numbers[0..12]: [285, 290, 289, 289, 281, 289, 289, 287, 281, 285, 290, 275]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.282044785Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236802023-09-22T23:35:06.337Z{"msg":"[0]R generation[0..12]: [176, 179, 178, 178, 174, 178, 178, 177, 174, 176, 179, 170]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.282098719Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236812023-09-22T23:35:06.340Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]"The guest has finished waiting for activation with:180
236822023-09-22T23:35:06.343ZWait for a query_work_queue command to finish before sending IO
236832023-09-22T23:35:06.347Z,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.282292115Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236842023-09-22T23:35:06.350Z{"msg":"[1]R flush_numbers[0..12]: [285, 290, 289, 289, 281, 289, 289, 287, 281, 285, 290, 275]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.282363151Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236852023-09-22T23:35:06.353Z{"msg":"[1]R generation[0..12]: [176, 179, 178, 178, 174, 178, 178, 177, 174, 176, 179, 170]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.282418641Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236862023-09-22T23:35:06.357Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.28246261Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236872023-09-22T23:35:06.360Z{"msg":"[2]R flush_numbers[0..12]: [285, 290, 289, 289, 281, 289, 289, 287, 281, 285, 290, 275]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.282484735Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236882023-09-22T23:35:06.363Z{"msg":"[2]R generation[0..12]: [176, 179, 178, 178, 174, 178, 178, 177, 174, 176, 179, 170]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.282510353Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236892023-09-22T23:35:06.366Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.282574642Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236902023-09-22T23:35:06.369Z{"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.282636985Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236912023-09-22T23:35:06.372Z{"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.282681638Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236922023-09-22T23:35:06.375Z{"msg":"Next flush: 291","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.282732576Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236932023-09-22T23:35:06.378Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.282796369Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236942023-09-22T23:35:06.381Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.282856128Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236952023-09-22T23:35:06.384Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.282900773Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236962023-09-22T23:35:06.387Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.282947481Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236972023-09-22T23:35:06.390Z{"msg":"c9cbfa6b-b641-4753-932e-e5ca1b09c279 is now active with session: c3fdb5c1-897c-4d6e-b241-8e01c7ed852e","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.282987214Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236982023-09-22T23:35:06.393Z{"msg":"c9cbfa6b-b641-4753-932e-e5ca1b09c279 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.283037236Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
236992023-09-22T23:35:06.396Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.283082864Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
237002023-09-22T23:35:06.399Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.283120913Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
237012023-09-22T23:35:06.401Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.283159927Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
237022023-09-22T23:35:06.404Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.283198079Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
237032023-09-22T23:35:06.407Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.283235891Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
237042023-09-22T23:35:06.410Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
237052023-09-22T23:35:06.413Z,"time":"2023-09-22T23:35:06.28327407Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
237062023-09-22T23:35:06.416Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.283296731Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
237072023-09-22T23:35:06.421Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.283332263Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
237082023-09-22T23:35:06.424Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.283370078Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
237092023-09-22T23:35:06.427Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.283408104Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
237102023-09-22T23:35:06.429Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.283445263Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1903}
237112023-09-22T23:35:06.432ZLoading write count information from file "/tmp/test_repair_verify.data"
237122023-09-22T23:35:06.435ZRun Repair workload
237132023-09-22T23:35:06.438Z01/30 Write block 468 len 32768 data: 17 16 14 12 13 12 12 11
237142023-09-22T23:35:06.441Z02/30 Read block 442 len 12288
237152023-09-22T23:35:06.444Z03/30 Write block 296 len 40960 data: 16 12 12 12 10 13 15 16 15 15
237162023-09-22T23:35:06.447Z04/30 Read block 434 len 8192
237172023-09-22T23:35:06.451Z05/30 Write block 349 len 16384 data: 13 14 16 16
237182023-09-22T23:35:06.454Z06/30 Read block 589 len 40960
237192023-09-22T23:35:06.457Z07/30 Read block 54 len 28672
237202023-09-22T23:35:06.460Z08/30 Write block 351 len 28672 data: 17 17 17 16 16 15 13
237212023-09-22T23:35:06.462Z09/30 Read block 162 len 20480
237222023-09-22T23:35:06.465Z10/30 Read block 87 len 36864
237232023-09-22T23:35:06.468Z11/30 Read block 583 len 40960
237242023-09-22T23:35:06.471Z12/30 Write block 241 len 32768 data: 18 15 11 10 11 7 10 10
237252023-09-22T23:35:06.475Z13/30 Write block 85 len 20480 data: 12 12 16 13 13
237262023-09-22T23:35:06.478Z14/30 Write block 31 len 8192 data: 11 8
237272023-09-22T23:35:06.481Z15/30 Flush
237282023-09-22T23:35:06.484Z16/30 Write block 32 len 24576 data: 9 8 12 13 16 16
237292023-09-22T23:35:06.487Z17/30 Write block 189 len 28672 data: 11 13 14 12 12 13 14
237302023-09-22T23:35:06.490Z18/30 Write block 410 len 4096 data: 11
237312023-09-22T23:35:06.493Z19/30 Read block 474 len 28672
237322023-09-22T23:35:06.496Z20/30 Read block 216 len 20480
237332023-09-22T23:35:06.499Z21/30 Flush
237342023-09-22T23:35:06.502Z22/30 Write block 373 len 16384 data: 8 9 8 9
237352023-09-22T23:35:06.505Z23/30 Read block 84 len 28672
237362023-09-22T23:35:06.508Z24/30 Write block 334 len 28672 data: 18 18 18 14 15 14 11
237372023-09-22T23:35:06.511Z25/30 Flush
237382023-09-22T23:35:06.514Z26/30 Write block 308 len 32768 data: 15 14 11 10 9 11 14 10
237392023-09-22T23:35:06.518Z27/30 Read block 313 len 36864
237402023-09-22T23:35:06.521Z28/30 Write block 74 len 20480 data: 15 17 16 17 17
237412023-09-22T23:35:06.524Z29/30 Read block 73 len 20480
237422023-09-22T23:35:06.527Z30/30 Write block 227 len 4096 data: 10
237432023-09-22T23:35:06.530Z----------------------------------------------------------------
237442023-09-22T23:35:06.533Z Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:30
237452023-09-22T23:35:06.536ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
237462023-09-22T23:35:06.539Z 1 Acked 1000 Write 8 Done Done Done false
237472023-09-22T23:35:06.542Z 2 Acked 1001 Read 3 Done Done Done false
237482023-09-22T23:35:06.545Z 3 Acked 1002 Write 10 Done Sent Done false
237492023-09-22T23:35:06.548Z 4 Acked 1003 Read 2 Done Sent Done false
237502023-09-22T23:35:06.551Z 5 Acked 1004 Write 4 Done Sent Done false
237512023-09-22T23:35:06.554Z 6 Acked 1005 Read 10 Done Sent Done false
237522023-09-22T23:35:06.559Z 7 Acked 1006 Read 7 Done Sent Done false
237532023-09-22T23:35:06.562Z 8 Acked 1007 Write 7 Done Sent Done false
237542023-09-22T23:35:06.564Z 9 Acked 1008 Read 5 Done Sent Done false
237552023-09-22T23:35:06.567Z 10 Acked 1009 Read 9 Done Sent Done false
237562023-09-22T23:35:06.570Z 11 Acked 1010 Read 10 Done Sent Done false
237572023-09-22T23:35:06.573Z 12 Acked 1011 Write 8 Done Sent Done false
237582023-09-22T23:35:06.575Z 13 Acked 1012 Write 5 Done Sent Done false
237592023-09-22T23:35:06.578Z 14 Acked 1013 Write 2 Done Sent Done false
237602023-09-22T23:35:06.590Z 15 Acked 1014 Flush 0 Done Sent Done false
237612023-09-22T23:35:06.593Z 16 Acked 1015 Write 6 Done Sent Done false
237622023-09-22T23:35:06.596Z 17 Acked 1016 Write 7 Done Sent Done false
237632023-09-22T23:35:06.599Z 18 Acked 1017 Write 1 Done Sent Done false
237642023-09-22T23:35:06.602Z 19 Acked 1018 Read 7 Done Sent Done false
237652023-09-22T23:35:06.604Z 20 Acked 1019 Read 5 Done Sent Done false
237662023-09-22T23:35:06.607Z 21 Acked 1020 Flush 0 Done Sent Done false
237672023-09-22T23:35:06.610Z 22 Acked 1021 Write 4 Done Sent Done false
237682023-09-22T23:35:06.613Z 23 Acked 1022 Read 7 Done Sent Done false
237692023-09-22T23:35:06.616Z 24 Acked 1023 Write 7 Done Sent Done false
237702023-09-22T23:35:06.619Z 25 Acked 1024 Flush 0 Done Sent Done false
237712023-09-22T23:35:06.622Z 26 Acked 1025 Write 8 Done Sent Done false
237722023-09-22T23:35:06.625Z 27 Acked 1026 Read 9 Done Sent Done false
237732023-09-22T23:35:06.628Z 28 Acked 1027 Write 5 Done Sent Done false
237742023-09-22T23:35:06.631Z 29 Acked 1028 Read 5 Done Sent Done false
237752023-09-22T23:35:06.634Z 30 Acked 1029 Write 1 Sent Sent Sent false
237762023-09-22T23:35:06.637Z STATES DS:0 DS:1 DS:2 TOTAL
237772023-09-22T23:35:06.640Z New 0 0 0 0
237782023-09-22T23:35:06.642Z Sent 1 28 1 30
237792023-09-22T23:35:06.645Z Done 29 2 29 60
237802023-09-22T23:35:06.648Z Skipped 0 0 0 0
237812023-09-22T23:35:06.651Z Error 0 0 0 0
237822023-09-22T23:35:06.654ZLast Flush: 1024 0 1024
237832023-09-22T23:35:06.657ZDownstairs last five completed:
237842023-09-22T23:35:06.660ZUpstairs last five completed: 30 29 28 27 26 25
237852023-09-22T23:35:06.663ZWrote out file "/tmp/test_repair_verify.data"
237862023-09-22T23:35:06.666Z
237872023-09-22T23:35:06.668ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
237882023-09-22T23:35:06.671Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.428474239Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1904}
237892023-09-22T23:35:06.674Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.428783133Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1904}
237902023-09-22T23:35:06.677Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.428806434Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1904}
237912023-09-22T23:35:06.679Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.452999167Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1904}
237922023-09-22T23:35:06.682Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.453038454Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1904}
237932023-09-22T23:35:06.685Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.45305804Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1904}
237942023-09-22T23:35:06.688Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.472346264Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1904}
237952023-09-22T23:35:06.691Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.472381946Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1904}
237962023-09-22T23:35:06.694ZError: Difference in extent metadata found!
237972023-09-22T23:35:06.697Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:06.472400902Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1904}
237982023-09-22T23:35:06.700ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
237992023-09-22T23:35:06.702Z 0 000-019 176 176 176 285 285 285 F F F
238002023-09-22T23:35:06.705Z 1 020-039 180 179 180 292 290 292 F F F <---
238012023-09-22T23:35:06.708Z 2 040-059 178 178 178 289 289 289 F F F
238022023-09-22T23:35:06.711Z 3 060-079 178 178 178 289 289 289 T F T <---
238032023-09-22T23:35:06.714Z 4 080-099 180 174 180 291 281 291 F F F <---
238042023-09-22T23:35:06.717Z 5 100-119 178 178 178 289 289 289 F F F
238052023-09-22T23:35:06.720Z 6 120-139 178 178 178 289 289 289 F F F
238062023-09-22T23:35:06.723Z 7 140-159 177 177 177 287 287 287 F F F
238072023-09-22T23:35:06.725Z 8 160-179 174 174 174 281 281 281 F F F
238082023-09-22T23:35:06.728Z 9 180-199 180 176 180 292 285 292 F F F <---
238092023-09-22T23:35:06.731Z 10 200-219 179 179 179 290 290 290 F F F
238102023-09-22T23:35:06.734Z 11 220-239 170 170 170 275 275 275 T F T <---
238112023-09-22T23:35:06.737Z 12 240-259 180 174 180 291 280 291 F F F <---
238122023-09-22T23:35:06.740Z 13 260-279 172 172 172 278 278 278 F F F
238132023-09-22T23:35:06.743Z 14 280-299 180 174 180 291 281 291 F F F <---
238142023-09-22T23:35:06.745Z 15 300-319 180 179 180 291 290 291 T F T <---
238152023-09-22T23:35:06.748Z 16 320-339 180 176 180 293 285 293 F F F <---
238162023-09-22T23:35:06.751Z 17 340-359 180 178 180 293 288 293 F F F <---
238172023-09-22T23:35:06.755Z 18 360-379 180 179 180 293 290 293 F F F <---
238182023-09-22T23:35:06.758Z 19 380-399 173 173 173 279 279 279 F F F
238192023-09-22T23:35:06.760Z 20 400-419 180 169 180 292 273 292 F F F <---
238202023-09-22T23:35:06.763Z 21 420-439 178 178 178 289 289 289 F F F
238212023-09-22T23:35:06.767Z 22 440-459 175 175 175 283 283 283 F F F
238222023-09-22T23:35:06.770Z 23 460-479 180 177 180 291 287 291 F T F <---
238232023-09-22T23:35:06.773Z 24 480-499 177 177 177 287 287 287 F F F
238242023-09-22T23:35:06.776Z 25 500-519 174 174 174 281 281 281 F F F
238252023-09-22T23:35:06.780Z 26 520-539 177 177 177 287 287 287 F F F
238262023-09-22T23:35:06.783Z 27 540-559 178 178 178 289 289 289 F F F
238272023-09-22T23:35:06.786Z 28 560-579 179 179 179 290 290 290 F F F
238282023-09-22T23:35:06.789Z 29 580-599 177 177 177 287 287 287 F F F
238292023-09-22T23:35:06.792ZMax gen: 180, Max flush: 293
238302023-09-22T23:35:06.795ZA Difference in extent metadata is expected here
238312023-09-22T23:35:06.797ZOn loop 90
238322023-09-22T23:35:06.800Z
238332023-09-22T23:35:06.803ZVerifying data now
238342023-09-22T23:35:12.901ZNo change after verify
238352023-09-22T23:35:12.904ZLoop: 90 Downstairs dump after verify (and repair):
238362023-09-22T23:35:12.919Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:12.917486965Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1909}
238372023-09-22T23:35:12.922Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:12.917898612Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1909}
238382023-09-22T23:35:12.925Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:12.917923895Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1909}
238392023-09-22T23:35:12.938Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:12.936805886Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1909}
238402023-09-22T23:35:12.941Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:12.936867187Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1909}
238412023-09-22T23:35:12.944Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:12.936889531Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1909}
238422023-09-22T23:35:12.949Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:12.948042521Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1909}
238432023-09-22T23:35:12.952Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:12.948094353Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1909}
238442023-09-22T23:35:12.955Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:12.948114794Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1909}
238452023-09-22T23:35:12.960ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
238462023-09-22T23:35:12.964Z 0 000-019 176 176 176 285 285 285 F F F
238472023-09-22T23:35:12.968Z 1 020-039 180 180 180 292 292 292 F F F
238482023-09-22T23:35:12.971Z 2 040-059 178 178 178 289 289 289 F F F
238492023-09-22T23:35:12.974Z 3 060-079 181 181 181 294 294 294 F F F
238502023-09-22T23:35:12.976Z 4 080-099 180 180 180 291 291 291 F F F
238512023-09-22T23:35:12.979Z 5 100-119 178 178 178 289 289 289 F F F
238522023-09-22T23:35:12.982Z 6 120-139 178 178 178 289 289 289 F F F
238532023-09-22T23:35:12.985Z 7 140-159 177 177 177 287 287 287 F F F
238542023-09-22T23:35:12.988Z 8 160-179 174 174 174 281 281 281 F F F
238552023-09-22T23:35:12.991Z 9 180-199 180 180 180 292 292 292 F F F
238562023-09-22T23:35:12.993Z 10 200-219 179 179 179 290 290 290 F F F
238572023-09-22T23:35:12.996Z 11 220-239 181 181 181 294 294 294 F F F
238582023-09-22T23:35:12.999Z 12 240-259 180 180 180 291 291 291 F F F
238592023-09-22T23:35:13.003Z 13 260-279 172 172 172 278 278 278 F F F
238602023-09-22T23:35:13.005Z 14 280-299 180 180 180 291 291 291 F F F
238612023-09-22T23:35:13.008Z 15 300-319 181 181 181 294 294 294 F F F
238622023-09-22T23:35:13.011Z 16 320-339 180 180 180 293 293 293 F F F
238632023-09-22T23:35:13.014Z 17 340-359 180 180 180 293 293 293 F F F
238642023-09-22T23:35:13.017Z 18 360-379 180 180 180 293 293 293 F F F
238652023-09-22T23:35:13.020Z 19 380-399 173 173 173 279 279 279 F F F
238662023-09-22T23:35:13.023Z 20 400-419 180 180 180 292 292 292 F F F
238672023-09-22T23:35:13.026Z 21 420-439 178 178 178 289 289 289 F F F
238682023-09-22T23:35:13.029Z 22 440-459 175 175 175 283 283 283 F F F
238692023-09-22T23:35:13.031Z 23 460-479 180 180 180 291 291 291 F F F
238702023-09-22T23:35:13.034Z 24 480-499 177 177 177 287 287 287 F F F
238712023-09-22T23:35:13.037Z 25 500-519 174 174 174 281 281 281 F F F
238722023-09-22T23:35:13.040Z 26 520-539 177 177 177 287 287 287 F F F
238732023-09-22T23:35:13.042Z 27 540-559 178 178 178 289 289 289 F F F
238742023-09-22T23:35:13.046Z 28 560-579 179 179 179 290 290 290 F F F
238752023-09-22T23:35:13.048Z 29 580-599 177 177 177 287 287 287 F F F
238762023-09-22T23:35:13.051ZMax gen: 181, Max flush: 294
238772023-09-22T23:35:13.054Z
238782023-09-22T23:35:13.057ZBegin loop 91
238792023-09-22T23:35:13.059ZDownstairs to restart: 2
238802023-09-22T23:35:13.062Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:12.997399674Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
238812023-09-22T23:35:13.065Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:12.997779865Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
238822023-09-22T23:35:13.068Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:12.997831674Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
238832023-09-22T23:35:13.071Z{"msg":"Crucible stats registered with UUID: 35d2e872-9bdb-4318-b7df-b720c4c5a8cf","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:12.997875211Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
238842023-09-22T23:35:13.074Z{"msg":"Crucible 35d2e872-9bdb-4318-b7df-b720c4c5a8cf has session id: 23d38e68-9ea0-4d2c-9520-d5c28bdc64cf","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:12.997933794Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
238852023-09-22T23:35:13.077ZCrucible runtime is spawned
238862023-09-22T23:35:13.079Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30The guest has requested activation with gen:182
238872023-09-22T23:35:13.082Z,"time":"2023-09-22T23:35:12.998237459Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911,"task":"up_listen"}
238882023-09-22T23:35:13.085Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:12.998290907Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
238892023-09-22T23:35:13.087Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:12.998357012Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
238902023-09-22T23:35:13.090Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:12.998410726Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911,"looper":"0"}
238912023-09-22T23:35:13.093Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:12.998669674Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911,"looper":"1"}
238922023-09-22T23:35:13.096Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:12.998734077Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911,"looper":"2"}
238932023-09-22T23:35:13.099Z{"msg":"[0] 35d2e872-9bdb-4318-b7df-b720c4c5a8cf looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:12.998807627Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911,"looper":"0"}
238942023-09-22T23:35:13.102Z{"msg":"[1] 35d2e872-9bdb-4318-b7df-b720c4c5a8cf looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:12.998839068Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911,"looper":"1"}
238952023-09-22T23:35:13.104Z{"msg":"35d2e872-9bdb-4318-b7df-b720c4c5a8cf active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:12.999175035Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
238962023-09-22T23:35:13.107Z{"msg":"Set desired generation to :182","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:12.999226419Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
238972023-09-22T23:35:13.110Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:12.999348467Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
238982023-09-22T23:35:13.112Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:12.999663259Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
238992023-09-22T23:35:13.115Z{"msg":"[0] 35d2e872-9bdb-4318-b7df-b720c4c5a8cf (2dc77d14-88df-4b61-a3a1-8586e294380c) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:13.000031041Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239002023-09-22T23:35:13.118Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:13.000079564Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239012023-09-22T23:35:13.121Z{"msg":"[0] client is_active_req TRUE, promote! session 2dc77d14-88df-4b61-a3a1-8586e294380c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:13.000102549Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239022023-09-22T23:35:13.124Z{"msg":"[1] 35d2e872-9bdb-4318-b7df-b720c4c5a8cf (2dc77d14-88df-4b61-a3a1-8586e294380c) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:13.000211735Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239032023-09-22T23:35:13.126Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:13.000303214Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239042023-09-22T23:35:13.129Z{"msg":"[1] client is_active_req TRUE, promote! session 2dc77d14-88df-4b61-a3a1-8586e294380c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:13.000388964Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239052023-09-22T23:35:13.132Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:13.000664141Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239062023-09-22T23:35:13.135Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:13.000777979Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239072023-09-22T23:35:13.138Z{"msg":"35d2e872-9bdb-4318-b7df-b720c4c5a8cf WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:13.001007784Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239082023-09-22T23:35:13.141Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:13.001067421Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239092023-09-22T23:35:13.143Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:13.001129848Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239102023-09-22T23:35:13.146Z{"msg":"35d2e872-9bdb-4318-b7df-b720c4c5a8cf WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:13.001183382Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239112023-09-22T23:35:13.149Z{"msg":"[0] 35d2e872-9bdb-4318-b7df-b720c4c5a8cf (2dc77d14-88df-4b61-a3a1-8586e294380c) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:13.002879945Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239122023-09-22T23:35:13.152Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:13.002963366Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239132023-09-22T23:35:13.155Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:35:13.003035418Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239142023-09-22T23:35:13.158Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:13.003109527Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239152023-09-22T23:35:13.161Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:13.003183866Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239162023-09-22T23:35:13.164Z{"msg":"35d2e872-9bdb-4318-b7df-b720c4c5a8cf WaitQuorum WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:13.003240502Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239172023-09-22T23:35:13.167Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:13.00328236Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239182023-09-22T23:35:13.169Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:13.00332178Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239192023-09-22T23:35:13.172Z{"msg":"[1] 35d2e872-9bdb-4318-b7df-b720c4c5a8cf (2dc77d14-88df-4b61-a3a1-8586e294380c) WaitQuorum WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:13.004041083Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239202023-09-22T23:35:13.175Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:13.00412286Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239212023-09-22T23:35:13.178Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:35:13.004194054Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239222023-09-22T23:35:13.181Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:13.004240036Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239232023-09-22T23:35:13.183Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:13.004280735Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239242023-09-22T23:35:13.186Z{"msg":"35d2e872-9bdb-4318-b7df-b720c4c5a8cf WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:13.004328975Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239252023-09-22T23:35:13.189Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:13.004368593Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239262023-09-22T23:35:13.191Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:13.004440059Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239272023-09-22T23:35:14.013Z{"msg":"[2] 35d2e872-9bdb-4318-b7df-b720c4c5a8cf looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.000384703Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911,"looper":"2"}
239282023-09-22T23:35:14.018Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.000470876Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239292023-09-22T23:35:14.021Z{"msg":"[2] 35d2e872-9bdb-4318-b7df-b720c4c5a8cf (2dc77d14-88df-4b61-a3a1-8586e294380c) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.001603335Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239302023-09-22T23:35:14.024Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.001656842Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239312023-09-22T23:35:14.027Z{"msg":"[2] client is_active_req TRUE, promote! session 2dc77d14-88df-4b61-a3a1-8586e294380c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.001700208Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239322023-09-22T23:35:14.030Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.00225747Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239332023-09-22T23:35:14.033Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.002315724Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239342023-09-22T23:35:14.036Z{"msg":"35d2e872-9bdb-4318-b7df-b720c4c5a8cf WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.002358363Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239352023-09-22T23:35:14.039Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.004518317Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239362023-09-22T23:35:14.042Z{"msg":"[2] 35d2e872-9bdb-4318-b7df-b720c4c5a8cf (2dc77d14-88df-4b61-a3a1-8586e294380c) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.004661065Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239372023-09-22T23:35:14.045Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.004770644Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239382023-09-22T23:35:14.048Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:35:14.004798688Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239392023-09-22T23:35:14.051Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.004847732Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239402023-09-22T23:35:14.054Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.004893961Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239412023-09-22T23:35:14.057Z{"msg":"35d2e872-9bdb-4318-b7df-b720c4c5a8cf WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.004934213Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239422023-09-22T23:35:14.060Z{"msg":"[0]R flush_numbers[0..12]: [285, 292, 289, 294, 291, 289, 289, 287, 281, 292, 290, 294]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.004974423Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239432023-09-22T23:35:14.063Z{"msg":"[0]R generation[0..12]: [176, 180, 178, 181, 180, 178, 178, 177, 174, 180, 179, 181]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.005019465Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239442023-09-22T23:35:14.066Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30The guest has finished waiting for activation with:182
239452023-09-22T23:35:14.069ZWait for a query_work_queue command to finish before sending IO
239462023-09-22T23:35:14.072Z,"time":"2023-09-22T23:35:14.005059361Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239472023-09-22T23:35:14.075Z{"msg":"[1]R flush_numbers[0..12]: [285, 292, 289, 294, 291, 289, 289, 287, 281, 292, 290, 294]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.00522303Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239482023-09-22T23:35:14.078Z{"msg":"[1]R generation[0..12]: [176, 180, 178, 181, 180, 178, 178, 177, 174, 180, 179, 181]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.005294629Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239492023-09-22T23:35:14.080Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.005366346Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239502023-09-22T23:35:14.083Z{"msg":"[2]R flush_numbers[0..12]: [285, 292, 289, 294, 291, 289, 289, 287, 281, 292, 290, 294]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.005432647Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239512023-09-22T23:35:14.086Z{"msg":"[2]R generation[0..12]: [176, 180, 178, 181, 180, 178, 178, 177, 174, 180, 179, 181]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.005505759Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239522023-09-22T23:35:14.089Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.005574524Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239532023-09-22T23:35:14.092Z{"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.00564467Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239542023-09-22T23:35:14.095Z{"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.00571489Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239552023-09-22T23:35:14.098Z{"msg":"Next flush: 295","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.005784901Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239562023-09-22T23:35:14.101Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.005876275Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239572023-09-22T23:35:14.104Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.005944672Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239582023-09-22T23:35:14.106Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.006009939Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239592023-09-22T23:35:14.110Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
239602023-09-22T23:35:14.113Z,"time":"2023-09-22T23:35:14.006076842Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239612023-09-22T23:35:14.116Z{"msg":"35d2e872-9bdb-4318-b7df-b720c4c5a8cf is now active with session: 2dc77d14-88df-4b61-a3a1-8586e294380c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.006190917Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239622023-09-22T23:35:14.119Z{"msg":"35d2e872-9bdb-4318-b7df-b720c4c5a8cf Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.006237704Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239632023-09-22T23:35:14.121Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.006277104Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239642023-09-22T23:35:14.125Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.006314697Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239652023-09-22T23:35:14.128Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.006352968Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239662023-09-22T23:35:14.132Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.006391882Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239672023-09-22T23:35:14.135Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.00642932Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239682023-09-22T23:35:14.137Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.006467041Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239692023-09-22T23:35:14.140Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.006504739Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239702023-09-22T23:35:14.149Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.006542458Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239712023-09-22T23:35:14.157Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.006580067Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239722023-09-22T23:35:14.160Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.006618038Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239732023-09-22T23:35:14.163Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.006656109Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1911}
239742023-09-22T23:35:14.167ZLoading write count information from file "/tmp/test_repair_verify.data"
239752023-09-22T23:35:14.170ZRun Repair workload
239762023-09-22T23:35:14.173Z01/30 Flush
239772023-09-22T23:35:14.176Z02/30 Flush
239782023-09-22T23:35:14.179Z03/30 Write block 382 len 36864 data: 12 12 12 12 10 15 15 15 15
239792023-09-22T23:35:14.186Z04/30 Read block 14 len 20480
239802023-09-22T23:35:14.191Z05/30 Flush
239812023-09-22T23:35:14.194Z06/30 Write block 512 len 24576 data: 16 18 21 20 18 18
239822023-09-22T23:35:14.199Z07/30 Write block 558 len 8192 data: 10 12
239832023-09-22T23:35:14.202Z08/30 Write block 272 len 16384 data: 21 19 17 18
239842023-09-22T23:35:14.206Z09/30 Read block 41 len 28672
239852023-09-22T23:35:14.209Z10/30 Flush
239862023-09-22T23:35:14.212Z11/30 Write block 510 len 20480 data: 15 13 17 19 22
239872023-09-22T23:35:14.215Z12/30 Read block 471 len 24576
239882023-09-22T23:35:14.218Z13/30 Flush
239892023-09-22T23:35:14.221Z14/30 Write block 421 len 36864 data: 15 12 13 12 8 12 14 15 14
239902023-09-22T23:35:14.224Z15/30 Read block 453 len 24576
239912023-09-22T23:35:14.227Z16/30 Write block 371 len 20480 data: 11 11 9 10 9
239922023-09-22T23:35:14.230Z17/30 Write block 344 len 12288 data: 11 14 15
239932023-09-22T23:35:14.255Z18/30 Flush
239942023-09-22T23:35:14.258Z19/30 Write block 149 len 16384 data: 15 15 15 14
239952023-09-22T23:35:14.261Z20/30 Flush
239962023-09-22T23:35:14.266Z21/30 Write block 270 len 40960 data: 17 20 22 20 18 19 22 19 21 23
239972023-09-22T23:35:14.269Z22/30 Write block 124 len 36864 data: 12 10 8 12 12 10 11 10 14
239982023-09-22T23:35:14.272Z23/30 Write block 582 len 4096 data: 12
239992023-09-22T23:35:14.275Z24/30 Read block 396 len 8192
240002023-09-22T23:35:14.278Z25/30 Write block 483 len 28672 data: 15 20 21 24 20 25 27
240012023-09-22T23:35:14.281Z26/30 Write block 187 len 4096 data: 14
240022023-09-22T23:35:14.283Z27/30 Write block 448 len 36864 data: 12 13 15 14 13 15 13 13 12
240032023-09-22T23:35:14.286Z28/30 Read block 379 len 36864
240042023-09-22T23:35:14.289Z29/30 Read block 161 len 28672
240052023-09-22T23:35:14.293Z30/30 Read block 99 len 12288
240062023-09-22T23:35:14.296Z----------------------------------------------------------------
240072023-09-22T23:35:14.299Z Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:28
240082023-09-22T23:35:14.302ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
240092023-09-22T23:35:14.313Z 3 Acked 1002 Write 9 Done Done Sent false
240102023-09-22T23:35:14.316Z 4 Acked 1003 Read 5 Done Done Sent false
240112023-09-22T23:35:14.319Z 5 Acked 1004 Flush 0 Done Done Sent false
240122023-09-22T23:35:14.322Z 6 Acked 1005 Write 6 Done Done Sent false
240132023-09-22T23:35:14.325Z 7 Acked 1006 Write 2 Done Done Sent false
240142023-09-22T23:35:14.328Z 8 Acked 1007 Write 4 Done Done Sent false
240152023-09-22T23:35:14.331Z 9 Acked 1008 Read 7 Done Done Sent false
240162023-09-22T23:35:14.334Z 10 Acked 1009 Flush 0 Done Done Sent false
240172023-09-22T23:35:14.337Z 11 Acked 1010 Write 5 Done Done Sent false
240182023-09-22T23:35:14.339Z 12 Acked 1011 Read 6 Done Done Sent false
240192023-09-22T23:35:14.342Z 13 Acked 1012 Flush 0 Done Done Sent false
240202023-09-22T23:35:14.345Z 14 Acked 1013 Write 9 Done Done Sent false
240212023-09-22T23:35:14.348Z 15 Acked 1014 Read 6 Done Done Sent false
240222023-09-22T23:35:14.351Z 16 Acked 1015 Write 5 Done Done Sent false
240232023-09-22T23:35:14.354Z 17 Acked 1016 Write 3 Done Done Sent false
240242023-09-22T23:35:14.358Z 18 Acked 1017 Flush 0 Done Done Sent false
240252023-09-22T23:35:14.361Z 19 Acked 1018 Write 4 Done Done Sent false
240262023-09-22T23:35:14.363Z 20 Acked 1019 Flush 0 Done Done Sent false
240272023-09-22T23:35:14.367Z 21 Acked 1020 Write 10 Done Done Sent false
240282023-09-22T23:35:14.370Z 22 Acked 1021 Write 9 Done Done Sent false
240292023-09-22T23:35:14.373Z 23 Acked 1022 Write 1 Done Done Sent false
240302023-09-22T23:35:14.376Z 24 Acked 1023 Read 2 Done Done Sent false
240312023-09-22T23:35:14.379Z 25 Acked 1024 Write 7 Done Done Sent false
240322023-09-22T23:35:14.382Z 26 Acked 1025 Write 1 Done Done Sent false
240332023-09-22T23:35:14.385Z 27 Acked 1026 Write 9 Done Done Sent false
240342023-09-22T23:35:14.388Z 28 Acked 1027 Read 9 Done Done Sent false
240352023-09-22T23:35:14.391Z 29 Acked 1028 Read 7 Done Done Sent false
240362023-09-22T23:35:14.394Z 30 Acked 1029 Read 3 Done Done Sent false
240372023-09-22T23:35:14.398Z STATES DS:0 DS:1 DS:2 TOTAL
240382023-09-22T23:35:14.401Z New 0 0 0 0
240392023-09-22T23:35:14.404Z Sent 0 0 28 28
240402023-09-22T23:35:14.408Z Done 28 28 0 56
240412023-09-22T23:35:14.411Z Skipped 0 0 0 0
240422023-09-22T23:35:14.414Z Error 0 0 0 0
240432023-09-22T23:35:14.417ZLast Flush: 1019 1019 1001
240442023-09-22T23:35:14.420ZDownstairs last five completed: 1001 1000
240452023-09-22T23:35:14.423ZUpstairs last five completed: 30 29 28 27 26 25
240462023-09-22T23:35:14.426ZWrote out file "/tmp/test_repair_verify.data"
240472023-09-22T23:35:14.429Z
240482023-09-22T23:35:14.432ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
240492023-09-22T23:35:14.435Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.142050573Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1912}
240502023-09-22T23:35:14.438Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.142372923Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1912}
240512023-09-22T23:35:14.441ZError: Difference in extent metadata found!
240522023-09-22T23:35:14.444Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.142395723Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1912}
240532023-09-22T23:35:14.447Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.171365492Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1912}
240542023-09-22T23:35:14.451Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.171399607Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1912}
240552023-09-22T23:35:14.454Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.171418578Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1912}
240562023-09-22T23:35:14.457Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.190841446Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1912}
240572023-09-22T23:35:14.459Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.190875284Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1912}
240582023-09-22T23:35:14.462Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:14.190894701Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1912}
240592023-09-22T23:35:14.465ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
240602023-09-22T23:35:14.468Z 0 000-019 176 176 176 285 285 285 F F F
240612023-09-22T23:35:14.470Z 1 020-039 180 180 180 292 292 292 F F F
240622023-09-22T23:35:14.473Z 2 040-059 178 178 178 289 289 289 F F F
240632023-09-22T23:35:14.476Z 3 060-079 181 181 181 294 294 294 F F F
240642023-09-22T23:35:14.479Z 4 080-099 180 180 180 291 291 291 F F F
240652023-09-22T23:35:14.481Z 5 100-119 178 178 178 289 289 289 F F F
240662023-09-22T23:35:14.484Z 6 120-139 178 178 178 289 289 289 T T F <---
240672023-09-22T23:35:14.487Z 7 140-159 182 182 177 301 301 287 F F F <---
240682023-09-22T23:35:14.490Z 8 160-179 174 174 174 281 281 281 F F F
240692023-09-22T23:35:14.493Z 9 180-199 180 180 180 292 292 292 T T F <---
240702023-09-22T23:35:14.495Z 10 200-219 179 179 179 290 290 290 F F F
240712023-09-22T23:35:14.498Z 11 220-239 181 181 181 294 294 294 F F F
240722023-09-22T23:35:14.501Z 12 240-259 180 180 180 291 291 291 F F F
240732023-09-22T23:35:14.505Z 13 260-279 182 182 172 298 298 278 T T F <---
240742023-09-22T23:35:14.508Z 14 280-299 180 180 180 291 291 291 F F F
240752023-09-22T23:35:14.511Z 15 300-319 181 181 181 294 294 294 F F F
240762023-09-22T23:35:14.515Z 16 320-339 180 180 180 293 293 293 F F F
240772023-09-22T23:35:14.518Z 17 340-359 182 182 180 300 300 293 F F F <---
240782023-09-22T23:35:14.522Z 18 360-379 182 182 180 300 300 293 F F F <---
240792023-09-22T23:35:14.526Z 19 380-399 182 182 173 297 297 279 F F F <---
240802023-09-22T23:35:14.528Z 20 400-419 180 180 180 292 292 292 F F F
240812023-09-22T23:35:14.531Z 21 420-439 182 182 178 300 300 289 F F F <---
240822023-09-22T23:35:14.534Z 22 440-459 175 175 175 283 283 283 T T F <---
240832023-09-22T23:35:14.537Z 23 460-479 180 180 180 291 291 291 F F F
240842023-09-22T23:35:14.540Z 24 480-499 177 177 177 287 287 287 T T F <---
240852023-09-22T23:35:14.543Z 25 500-519 182 182 174 299 299 281 F F F <---
240862023-09-22T23:35:14.546Z 26 520-539 177 177 177 287 287 287 F F F
240872023-09-22T23:35:14.549Z 27 540-559 182 182 178 298 298 289 F F F <---
240882023-09-22T23:35:14.552Z 28 560-579 179 179 179 290 290 290 F F F
240892023-09-22T23:35:14.555Z 29 580-599 177 177 177 287 287 287 T T F <---
240902023-09-22T23:35:14.558ZMax gen: 182, Max flush: 301
240912023-09-22T23:35:14.561ZA Difference in extent metadata is expected here
240922023-09-22T23:35:14.564ZOn loop 91
240932023-09-22T23:35:14.567Z
240942023-09-22T23:35:14.571ZVerifying data now
240952023-09-22T23:35:20.624ZNo change after verify
240962023-09-22T23:35:20.627ZLoop: 91 Downstairs dump after verify (and repair):
240972023-09-22T23:35:20.642Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.640854153Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1917}
240982023-09-22T23:35:20.646Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.641249439Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1917}
240992023-09-22T23:35:20.649Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.64127402Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1917}
241002023-09-22T23:35:20.661Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.659788627Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1917}
241012023-09-22T23:35:20.664Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.659841008Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1917}
241022023-09-22T23:35:20.667Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.659861763Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1917}
241032023-09-22T23:35:20.672Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.670776681Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1917}
241042023-09-22T23:35:20.675Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.670841335Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1917}
241052023-09-22T23:35:20.678Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.670864228Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1917}
241062023-09-22T23:35:20.683ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
241072023-09-22T23:35:20.686Z 0 000-019 176 176 176 285 285 285 F F F
241082023-09-22T23:35:20.689Z 1 020-039 180 180 180 292 292 292 F F F
241092023-09-22T23:35:20.692Z 2 040-059 178 178 178 289 289 289 F F F
241102023-09-22T23:35:20.696Z 3 060-079 181 181 181 294 294 294 F F F
241112023-09-22T23:35:20.699Z 4 080-099 180 180 180 291 291 291 F F F
241122023-09-22T23:35:20.701Z 5 100-119 178 178 178 289 289 289 F F F
241132023-09-22T23:35:20.704Z 6 120-139 183 183 183 302 302 302 F F F
241142023-09-22T23:35:20.707Z 7 140-159 182 182 182 301 301 301 F F F
241152023-09-22T23:35:20.710Z 8 160-179 174 174 174 281 281 281 F F F
241162023-09-22T23:35:20.713Z 9 180-199 183 183 183 302 302 302 F F F
241172023-09-22T23:35:20.716Z 10 200-219 179 179 179 290 290 290 F F F
241182023-09-22T23:35:20.719Z 11 220-239 181 181 181 294 294 294 F F F
241192023-09-22T23:35:20.722Z 12 240-259 180 180 180 291 291 291 F F F
241202023-09-22T23:35:20.726Z 13 260-279 183 183 183 302 302 302 F F F
241212023-09-22T23:35:20.729Z 14 280-299 180 180 180 291 291 291 F F F
241222023-09-22T23:35:20.732Z 15 300-319 181 181 181 294 294 294 F F F
241232023-09-22T23:35:20.735Z 16 320-339 180 180 180 293 293 293 F F F
241242023-09-22T23:35:20.737Z 17 340-359 182 182 182 300 300 300 F F F
241252023-09-22T23:35:20.740Z 18 360-379 182 182 182 300 300 300 F F F
241262023-09-22T23:35:20.743Z 19 380-399 182 182 182 297 297 297 F F F
241272023-09-22T23:35:20.746Z 20 400-419 180 180 180 292 292 292 F F F
241282023-09-22T23:35:20.748Z 21 420-439 182 182 182 300 300 300 F F F
241292023-09-22T23:35:20.752Z 22 440-459 183 183 183 302 302 302 F F F
241302023-09-22T23:35:20.755Z 23 460-479 180 180 180 291 291 291 F F F
241312023-09-22T23:35:20.758Z 24 480-499 183 183 183 302 302 302 F F F
241322023-09-22T23:35:20.760Z 25 500-519 182 182 182 299 299 299 F F F
241332023-09-22T23:35:20.763Z 26 520-539 177 177 177 287 287 287 F F F
241342023-09-22T23:35:20.766Z 27 540-559 182 182 182 298 298 298 F F F
241352023-09-22T23:35:20.769Z 28 560-579 179 179 179 290 290 290 F F F
241362023-09-22T23:35:20.772Z 29 580-599 183 183 183 302 302 302 F F F
241372023-09-22T23:35:20.775ZMax gen: 183, Max flush: 302
241382023-09-22T23:35:20.778Z
241392023-09-22T23:35:20.781ZBegin loop 92
241402023-09-22T23:35:20.783ZDownstairs to restart: 1
241412023-09-22T23:35:20.786Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.718903371Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241422023-09-22T23:35:20.789Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.719310868Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241432023-09-22T23:35:20.792Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.719388296Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241442023-09-22T23:35:20.795Z{"msg":"Crucible stats registered with UUID: 96d802f7-d504-4ece-913f-df6cb267966d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.719426732Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241452023-09-22T23:35:20.798Z{"msg":"Crucible 96d802f7-d504-4ece-913f-df6cb267966d has session id: 3cf39df6-9109-4720-978c-5d2f21812f70","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.719493437Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241462023-09-22T23:35:20.800ZCrucible runtime is spawned
241472023-09-22T23:35:20.803Z{"msg":"up_listen starts","v":0,"name":"crucible","level"The guest has requested activation with gen:184
241482023-09-22T23:35:20.806Z:30,"time":"2023-09-22T23:35:20.719714279Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919,"task":"up_listen"}
241492023-09-22T23:35:20.809Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.719783849Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241502023-09-22T23:35:20.812Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.719840173Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241512023-09-22T23:35:20.815Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.719895534Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919,"looper":"0"}
241522023-09-22T23:35:20.818Z{"msg":"[0] 96d802f7-d504-4ece-913f-df6cb267966d looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.720270031Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919,"looper":"0"}
241532023-09-22T23:35:20.823Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.720331461Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919,"looper":"1"}
241542023-09-22T23:35:20.827Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.720515954Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919,"looper":"2"}
241552023-09-22T23:35:20.830Z{"msg":"[2] 96d802f7-d504-4ece-913f-df6cb267966d looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.720768427Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919,"looper":"2"}
241562023-09-22T23:35:20.833Z{"msg":"96d802f7-d504-4ece-913f-df6cb267966d active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.721117967Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241572023-09-22T23:35:20.836Z{"msg":"Set desired generation to :184","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.721194613Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241582023-09-22T23:35:20.839Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.721321701Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241592023-09-22T23:35:20.842Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.721696462Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241602023-09-22T23:35:20.845Z{"msg":"[0] 96d802f7-d504-4ece-913f-df6cb267966d (4b11aa7a-62c1-4b60-9ef6-ee68a8758f32) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.721996387Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241612023-09-22T23:35:20.848Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.722065388Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241622023-09-22T23:35:20.851Z{"msg":"[0] client is_active_req TRUE, promote! session 4b11aa7a-62c1-4b60-9ef6-ee68a8758f32","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.722125504Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241632023-09-22T23:35:20.854Z{"msg":"[2] 96d802f7-d504-4ece-913f-df6cb267966d (4b11aa7a-62c1-4b60-9ef6-ee68a8758f32) WaitActive New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.72218915Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241642023-09-22T23:35:20.856Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.722244656Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241652023-09-22T23:35:20.859Z{"msg":"[2] client is_active_req TRUE, promote! session 4b11aa7a-62c1-4b60-9ef6-ee68a8758f32","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.722301512Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241662023-09-22T23:35:20.862Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.722701144Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241672023-09-22T23:35:20.865Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.722775659Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241682023-09-22T23:35:20.868Z{"msg":"96d802f7-d504-4ece-913f-df6cb267966d WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.722833404Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241692023-09-22T23:35:20.871Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.722904504Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241702023-09-22T23:35:20.875Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.722974085Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241712023-09-22T23:35:20.878Z{"msg":"96d802f7-d504-4ece-913f-df6cb267966d WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.723028603Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241722023-09-22T23:35:20.881Z{"msg":"[0] 96d802f7-d504-4ece-913f-df6cb267966d (4b11aa7a-62c1-4b60-9ef6-ee68a8758f32) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.724971399Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241732023-09-22T23:35:20.885Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.725035415Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241742023-09-22T23:35:20.888Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:35:20.725093319Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241752023-09-22T23:35:20.891Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.725147299Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241762023-09-22T23:35:20.894Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.725206923Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241772023-09-22T23:35:20.897Z{"msg":"96d802f7-d504-4ece-913f-df6cb267966d WaitQuorum New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.72533845Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241782023-09-22T23:35:20.900Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.725402211Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241792023-09-22T23:35:20.903Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.725458104Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241802023-09-22T23:35:20.906Z{"msg":"[2] 96d802f7-d504-4ece-913f-df6cb267966d (4b11aa7a-62c1-4b60-9ef6-ee68a8758f32) WaitQuorum New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.725517974Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241812023-09-22T23:35:20.909Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.725572353Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241822023-09-22T23:35:20.912Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:35:20.725626358Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241832023-09-22T23:35:20.915Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.725679476Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241842023-09-22T23:35:20.918Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.725733676Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241852023-09-22T23:35:20.920Z{"msg":"96d802f7-d504-4ece-913f-df6cb267966d WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.725787654Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241862023-09-22T23:35:20.923Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.725841002Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241872023-09-22T23:35:20.926Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:20.725870372Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241882023-09-22T23:35:21.724Z{"msg":"[1] 96d802f7-d504-4ece-913f-df6cb267966d looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.722582446Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919,"looper":"1"}
241892023-09-22T23:35:21.728Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.722665678Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241902023-09-22T23:35:21.731Z{"msg":"[1] 96d802f7-d504-4ece-913f-df6cb267966d (4b11aa7a-62c1-4b60-9ef6-ee68a8758f32) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.723698329Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241912023-09-22T23:35:21.734Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.723753961Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241922023-09-22T23:35:21.737Z{"msg":"[1] client is_active_req TRUE, promote! session 4b11aa7a-62c1-4b60-9ef6-ee68a8758f32","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.723797202Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241932023-09-22T23:35:21.741Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.724408771Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241942023-09-22T23:35:21.744Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.724469074Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241952023-09-22T23:35:21.747Z{"msg":"96d802f7-d504-4ece-913f-df6cb267966d WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.724493618Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241962023-09-22T23:35:21.750Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.72668791Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241972023-09-22T23:35:21.753Z{"msg":"[1] 96d802f7-d504-4ece-913f-df6cb267966d (4b11aa7a-62c1-4b60-9ef6-ee68a8758f32) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.727027392Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241982023-09-22T23:35:21.756Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.727106318Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
241992023-09-22T23:35:21.759Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:35:21.727180351Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242002023-09-22T23:35:21.765Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.727247998Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242012023-09-22T23:35:21.767Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.727316117Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242022023-09-22T23:35:21.771Z{"msg":"96d802f7-d504-4ece-913f-df6cb267966d WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.727384757Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242032023-09-22T23:35:21.774Z{"msg":"[0]R flush_numbers[0..12]: [285, 292, 289, 294, 291, 289, 302, 301, 281, 302, 290, 294]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.727452183Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242042023-09-22T23:35:21.777Z{"msg":"[0]R generation[0..12]: [176, 180, 178, 181, 180, 178, 183, 182, 174, 183, 179, 181]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.727522509Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242052023-09-22T23:35:21.780Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.727654839Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242062023-09-22T23:35:21.783Z{"msg":"[1]R flush_numbers[0..12]: [285, 292, 289, 294, 291, 289, 302, 301, 281, 302, 290, 294]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.727764916Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242072023-09-22T23:35:21.787ZThe guest has finished waiting for activation with:184
242082023-09-22T23:35:21.790Z{"msg":"Wait for a query_work_queue command to finish before sending IO
242092023-09-22T23:35:21.793Z[1]R generation[0..12]: [176, 180, 178, 181, 180, 178, 183, 182, 174, 183, 179, 181]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.727843645Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242102023-09-22T23:35:21.796Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.727908169Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242112023-09-22T23:35:21.799Z{"msg":"[2]R flush_numbers[0..12]: [285, 292, 289, 294, 291, 289, 302, 301, 281, 302, 290, 294]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.727955697Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242122023-09-22T23:35:21.803Z{"msg":"[2]R generation[0..12]: [176, 180, 178, 181, 180, 178, 183, 182, 174, 183, 179, 181]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.72802239Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242132023-09-22T23:35:21.807Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.728079456Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242142023-09-22T23:35:21.809Z{"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.728200926Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242152023-09-22T23:35:21.812Z{"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.728289061Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242162023-09-22T23:35:21.815Z{"msg":"Next flush: 303","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.728360672Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242172023-09-22T23:35:21.817Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.728425104Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242182023-09-22T23:35:21.821Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.72847682Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242192023-09-22T23:35:21.824Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.728523799Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242202023-09-22T23:35:21.827Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.728562999Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242212023-09-22T23:35:21.830Z{"msg":"96d802f7-d504-4ece-913f-df6cb267966d is now active with session: 4b11aa7a-62c1-4b60-9ef6-ee68a8758f32","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.728612807Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242222023-09-22T23:35:21.833Z{"msg":"96d802f7-d504-4ece-913f-df6cb267966d Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.728657397Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242232023-09-22T23:35:21.836Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.728679634Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242242023-09-22T23:35:21.840Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.728728534Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242252023-09-22T23:35:21.843Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.72876915Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242262023-09-22T23:35:21.846Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.728790043Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242272023-09-22T23:35:21.849Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.72883398Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242282023-09-22T23:35:21.852Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.728879546Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242292023-09-22T23:35:21.858Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.728925545Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
242302023-09-22T23:35:21.862Zpid":1919}
242312023-09-22T23:35:21.866Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.729044353Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242322023-09-22T23:35:21.869Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.729090033Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242332023-09-22T23:35:21.872Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.729129105Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242342023-09-22T23:35:21.875Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.729166677Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1919}
242352023-09-22T23:35:21.878ZLoading write count information from file "/tmp/test_repair_verify.data"
242362023-09-22T23:35:21.881ZRun Repair workload
242372023-09-22T23:35:21.883Z01/30 Flush
242382023-09-22T23:35:21.886Z02/30 Write block 168 len 40960 data: 9 7 11 10 14 15 15 18 21 19
242392023-09-22T23:35:21.889Z03/30 Read block 276 len 20480
242402023-09-22T23:35:21.892Z04/30 Read block 217 len 4096
242412023-09-22T23:35:21.895Z05/30 Flush
242422023-09-22T23:35:21.898Z06/30 Write block 337 len 32768 data: 15 16 15 12 9 13 11 12
242432023-09-22T23:35:21.901Z07/30 Read block 582 len 16384
242442023-09-22T23:35:21.904Z08/30 Read block 221 len 16384
242452023-09-22T23:35:21.908Z09/30 Write block 292 len 36864 data: 23 19 17 17 17 13 13 13 11
242462023-09-22T23:35:21.911Z10/30 Write block 133 len 4096 data: 13
242472023-09-22T23:35:21.914Z11/30 Write block 250 len 16384 data: 10 9 13 12
242482023-09-22T23:35:21.917Z12/30 Flush
242492023-09-22T23:35:21.920Z13/30 Write block 408 len 24576 data: 10 12 12 15 12 11
242502023-09-22T23:35:21.923Z14/30 Write block 431 len 16384 data: 16 18 18 16
242512023-09-22T23:35:21.926Z15/30 Flush
242522023-09-22T23:35:21.929Z16/30 Write block 236 len 20480 data: 12 12 12 14 17
242532023-09-22T23:35:21.932Z17/30 Read block 94 len 24576
242542023-09-22T23:35:21.935Z18/30 Write block 523 len 20480 data: 14 15 17 17 19
242552023-09-22T23:35:21.938Z19/30 Read block 280 len 4096
242562023-09-22T23:35:21.942Z20/30 Write block 109 len 32768 data: 21 21 18 18 18 16 13 13
242572023-09-22T23:35:21.945Z21/30 Read block 128 len 36864
242582023-09-22T23:35:21.947Z22/30 Write block 308 len 20480 data: 16 15 12 11 10
242592023-09-22T23:35:21.950Z23/30 Flush
242602023-09-22T23:35:21.953Z24/30 Write block 466 len 4096 data: 16
242612023-09-22T23:35:21.956Z25/30 Write block 18 len 16384 data: 13 12 9 7
242622023-09-22T23:35:21.959Z26/30 Flush
242632023-09-22T23:35:21.962Z27/30 Write block 110 len 20480 data: 22 19 19 19 17
242642023-09-22T23:35:21.965Z28/30 Read block 289 len 4096
242652023-09-22T23:35:21.968Z29/30 Write block 474 len 12288 data: 13 12 11
242662023-09-22T23:35:21.971Z30/30 Read block 492 len 20480
242672023-09-22T23:35:21.974Z----------------------------------------------------------------
242682023-09-22T23:35:21.977Z Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:29
242692023-09-22T23:35:21.980ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
242702023-09-22T23:35:21.983Z 2 Acked 1001 Write 10 Done Done Done false
242712023-09-22T23:35:21.986Z 3 Acked 1002 Read 5 Done Sent Done false
242722023-09-22T23:35:21.989Z 4 Acked 1003 Read 1 Done Sent Done false
242732023-09-22T23:35:21.991Z 5 Acked 1004 Flush 0 Done Sent Done false
242742023-09-22T23:35:21.994Z 6 Acked 1005 Write 8 Done Sent Done false
242752023-09-22T23:35:21.997Z 7 Acked 1006 Read 4 Done Sent Done false
242762023-09-22T23:35:22.001Z 8 Acked 1007 Read 4 Done Sent Done false
242772023-09-22T23:35:22.003Z 9 Acked 1008 Write 9 Done Sent Done false
242782023-09-22T23:35:22.006Z 10 Acked 1009 Write 1 Done Sent Done false
242792023-09-22T23:35:22.009Z 11 Acked 1010 Write 4 Done Sent Done false
242802023-09-22T23:35:22.012Z 12 Acked 1011 Flush 0 Done Sent Done false
242812023-09-22T23:35:22.015Z 13 Acked 1012 Write 6 Done Sent Done false
242822023-09-22T23:35:22.018Z 14 Acked 1013 Write 4 Done Sent Done false
242832023-09-22T23:35:22.021Z 15 Acked 1014 Flush 0 Done Sent Done false
242842023-09-22T23:35:22.024Z 16 Acked 1015 Write 5 Done Sent Done false
242852023-09-22T23:35:22.027Z 17 Acked 1016 Read 6 Done Sent Done false
242862023-09-22T23:35:22.030Z 18 Acked 1017 Write 5 Done Sent Done false
242872023-09-22T23:35:22.033Z 19 Acked 1018 Read 1 Done Sent Done false
242882023-09-22T23:35:22.036Z 20 Acked 1019 Write 8 Done Sent Done false
242892023-09-22T23:35:22.039Z 21 Acked 1020 Read 9 Done Sent Done false
242902023-09-22T23:35:22.042Z 22 Acked 1021 Write 5 Done Sent Done false
242912023-09-22T23:35:22.045Z 23 Acked 1022 Flush 0 Done Sent Done false
242922023-09-22T23:35:22.048Z 24 Acked 1023 Write 1 Done Sent Done false
242932023-09-22T23:35:22.054Z 25 Acked 1024 Write 4 Done Sent Done false
242942023-09-22T23:35:22.057Z 26 Acked 1025 Flush 0 Done Sent Done false
242952023-09-22T23:35:22.060Z 27 Acked 1026 Write 5 Done Sent Done false
242962023-09-22T23:35:22.062Z 28 Acked 1027 Read 1 Done Sent Done false
242972023-09-22T23:35:22.065Z 29 Acked 1028 Write 3 Done Sent Done false
242982023-09-22T23:35:22.068Z 30 Acked 1029 Read 5 Done Sent Done false
242992023-09-22T23:35:22.071Z STATES DS:0 DS:1 DS:2 TOTAL
243002023-09-22T23:35:22.074Z New 0 0 0 0
243012023-09-22T23:35:22.077Z Sent 0 28 0 28
243022023-09-22T23:35:22.079Z Done 29 1 29 59
243032023-09-22T23:35:22.082Z Skipped 0 0 0 0
243042023-09-22T23:35:22.085Z Error 0 0 0 0
243052023-09-22T23:35:22.088ZLast Flush: 1025 1000 1025
243062023-09-22T23:35:22.091ZDownstairs last five completed: 1000
243072023-09-22T23:35:22.094ZUpstairs last five completed: 30 29 28 27 26 25
243082023-09-22T23:35:22.098ZWrote out file "/tmp/test_repair_verify.data"
243092023-09-22T23:35:22.101Z
243102023-09-22T23:35:22.104ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
243112023-09-22T23:35:22.107Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.873798426Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1920}
243122023-09-22T23:35:22.110Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.874058668Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1920}
243132023-09-22T23:35:22.113Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.874081527Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1920}
243142023-09-22T23:35:22.115Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.897320391Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1920}
243152023-09-22T23:35:22.118Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.897353428Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1920}
243162023-09-22T23:35:22.121Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.897372166Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1920}
243172023-09-22T23:35:22.124Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.916667373Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1920}
243182023-09-22T23:35:22.127Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.916698827Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1920}
243192023-09-22T23:35:22.131ZError: Difference in extent metadata found!
243202023-09-22T23:35:22.134Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:21.916717644Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1920}
243212023-09-22T23:35:22.137ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
243222023-09-22T23:35:22.140Z 0 000-019 184 176 184 308 285 308 F F F <---
243232023-09-22T23:35:22.143Z 1 020-039 184 180 184 308 292 308 F F F <---
243242023-09-22T23:35:22.146Z 2 040-059 178 178 178 289 289 289 F F F
243252023-09-22T23:35:22.149Z 3 060-079 181 181 181 294 294 294 F F F
243262023-09-22T23:35:22.152Z 4 080-099 180 180 180 291 291 291 F F F
243272023-09-22T23:35:22.158Z 5 100-119 184 178 184 307 289 307 T F T <---
243282023-09-22T23:35:22.163Z 6 120-139 184 183 184 305 302 305 F F F <---
243292023-09-22T23:35:22.166Z 7 140-159 182 182 182 301 301 301 F F F
243302023-09-22T23:35:22.168Z 8 160-179 184 174 184 304 281 304 F T F <---
243312023-09-22T23:35:22.171Z 9 180-199 183 183 183 302 302 302 F F F
243322023-09-22T23:35:22.175Z 10 200-219 179 179 179 290 290 290 F F F
243332023-09-22T23:35:22.177Z 11 220-239 184 181 184 307 294 307 F F F <---
243342023-09-22T23:35:22.182Z 12 240-259 184 180 184 307 291 307 F F F <---
243352023-09-22T23:35:22.185Z 13 260-279 183 183 183 302 302 302 F F F
243362023-09-22T23:35:22.188Z 14 280-299 184 180 184 305 291 305 F F F <---
243372023-09-22T23:35:22.190Z 15 300-319 184 181 184 307 294 307 F F F <---
243382023-09-22T23:35:22.193Z 16 320-339 184 180 184 305 293 305 F F F <---
243392023-09-22T23:35:22.196Z 17 340-359 184 182 184 305 300 305 F F F <---
243402023-09-22T23:35:22.200Z 18 360-379 182 182 182 300 300 300 F F F
243412023-09-22T23:35:22.203Z 19 380-399 182 182 182 297 297 297 F F F
243422023-09-22T23:35:22.206Z 20 400-419 184 180 184 306 292 306 F F F <---
243432023-09-22T23:35:22.209Z 21 420-439 184 182 184 306 300 306 F F F <---
243442023-09-22T23:35:22.212Z 22 440-459 183 183 183 302 302 302 F F F
243452023-09-22T23:35:22.215Z 23 460-479 184 180 184 308 291 308 T F T <---
243462023-09-22T23:35:22.218Z 24 480-499 183 183 183 302 302 302 F F F
243472023-09-22T23:35:22.221Z 25 500-519 182 182 182 299 299 299 F F F
243482023-09-22T23:35:22.224Z 26 520-539 184 177 184 307 287 307 F F F <---
243492023-09-22T23:35:22.227Z 27 540-559 182 182 182 298 298 298 F F F
243502023-09-22T23:35:22.230Z 28 560-579 179 179 179 290 290 290 F F F
243512023-09-22T23:35:22.233Z 29 580-599 183 183 183 302 302 302 F F F
243522023-09-22T23:35:22.236ZMax gen: 184, Max flush: 308
243532023-09-22T23:35:22.238ZA Difference in extent metadata is expected here
243542023-09-22T23:35:22.241ZOn loop 92
243552023-09-22T23:35:22.244Z
243562023-09-22T23:35:22.247ZVerifying data now
243572023-09-22T23:35:28.499ZNo change after verify
243582023-09-22T23:35:28.502ZLoop: 92 Downstairs dump after verify (and repair):
243592023-09-22T23:35:28.517Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.515885827Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1925}
243602023-09-22T23:35:28.520Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.516281895Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1925}
243612023-09-22T23:35:28.523Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.516306357Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1925}
243622023-09-22T23:35:28.536Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.535256555Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1925}
243632023-09-22T23:35:28.539Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.53530926Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1925}
243642023-09-22T23:35:28.542Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.53532998Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1925}
243652023-09-22T23:35:28.548Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.546438341Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1925}
243662023-09-22T23:35:28.571Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.546489797Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1925}
243672023-09-22T23:35:28.574Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.546510035Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1925}
243682023-09-22T23:35:28.577ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
243692023-09-22T23:35:28.580Z 0 000-019 184 184 184 308 308 308 F F F
243702023-09-22T23:35:28.582Z 1 020-039 184 184 184 308 308 308 F F F
243712023-09-22T23:35:28.585Z 2 040-059 178 178 178 289 289 289 F F F
243722023-09-22T23:35:28.588Z 3 060-079 181 181 181 294 294 294 F F F
243732023-09-22T23:35:28.591Z 4 080-099 180 180 180 291 291 291 F F F
243742023-09-22T23:35:28.593Z 5 100-119 185 185 185 309 309 309 F F F
243752023-09-22T23:35:28.597Z 6 120-139 184 184 184 305 305 305 F F F
243762023-09-22T23:35:28.599Z 7 140-159 182 182 182 301 301 301 F F F
243772023-09-22T23:35:28.602Z 8 160-179 184 184 184 304 304 304 F F F
243782023-09-22T23:35:28.605Z 9 180-199 183 183 183 302 302 302 F F F
243792023-09-22T23:35:28.608Z 10 200-219 179 179 179 290 290 290 F F F
243802023-09-22T23:35:28.611Z 11 220-239 184 184 184 307 307 307 F F F
243812023-09-22T23:35:28.614Z 12 240-259 184 184 184 307 307 307 F F F
243822023-09-22T23:35:28.617Z 13 260-279 183 183 183 302 302 302 F F F
243832023-09-22T23:35:28.620Z 14 280-299 184 184 184 305 305 305 F F F
243842023-09-22T23:35:28.623Z 15 300-319 184 184 184 307 307 307 F F F
243852023-09-22T23:35:28.625Z 16 320-339 184 184 184 305 305 305 F F F
243862023-09-22T23:35:28.628Z 17 340-359 184 184 184 305 305 305 F F F
243872023-09-22T23:35:28.631Z 18 360-379 182 182 182 300 300 300 F F F
243882023-09-22T23:35:28.633Z 19 380-399 182 182 182 297 297 297 F F F
243892023-09-22T23:35:28.636Z 20 400-419 184 184 184 306 306 306 F F F
243902023-09-22T23:35:28.639Z 21 420-439 184 184 184 306 306 306 F F F
243912023-09-22T23:35:28.642Z 22 440-459 183 183 183 302 302 302 F F F
243922023-09-22T23:35:28.645Z 23 460-479 185 185 185 309 309 309 F F F
243932023-09-22T23:35:28.648Z 24 480-499 183 183 183 302 302 302 F F F
243942023-09-22T23:35:28.651Z 25 500-519 182 182 182 299 299 299 F F F
243952023-09-22T23:35:28.653Z 26 520-539 184 184 184 307 307 307 F F F
243962023-09-22T23:35:28.656Z 27 540-559 182 182 182 298 298 298 F F F
243972023-09-22T23:35:28.658Z 28 560-579 179 179 179 290 290 290 F F F
243982023-09-22T23:35:28.661Z 29 580-599 183 183 183 302 302 302 F F F
243992023-09-22T23:35:28.664ZMax gen: 185, Max flush: 309
244002023-09-22T23:35:28.667Z
244012023-09-22T23:35:28.670ZBegin loop 93
244022023-09-22T23:35:28.673ZDownstairs to restart: 1
244032023-09-22T23:35:28.675Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.594542739Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244042023-09-22T23:35:28.678Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.594998131Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244052023-09-22T23:35:28.684Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.595051015Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244062023-09-22T23:35:28.687Z{"msg":"Crucible stats registered with UUID: 8e3bcd14-e0d7-4242-b633-40317286a464","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.595095307Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244072023-09-22T23:35:28.694Z{"msg":"Crucible 8e3bcd14-e0d7-4242-b633-40317286a464 has session id: 8e4fba9e-2835-4631-9d82-dcc2842aba38","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.59513767Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244082023-09-22T23:35:28.697ZCrucible runtime is spawned
244092023-09-22T23:35:28.700ZThe guest has requested activation with gen:186
244102023-09-22T23:35:28.703Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.595738153Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927,"task":"up_listen"}
244112023-09-22T23:35:28.706Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.595832448Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244122023-09-22T23:35:28.708Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.595889378Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927,"looper":"1"}
244132023-09-22T23:35:28.711Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.595963611Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244142023-09-22T23:35:28.714Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.596027033Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927,"looper":"0"}
244152023-09-22T23:35:28.716Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.596082701Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927,"looper":"2"}
244162023-09-22T23:35:28.719Z{"msg":"[0] 8e3bcd14-e0d7-4242-b633-40317286a464 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.596139299Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927,"looper":"0"}
244172023-09-22T23:35:28.722Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.596221891Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244182023-09-22T23:35:28.725Z{"msg":"8e3bcd14-e0d7-4242-b633-40317286a464 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.596286701Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244192023-09-22T23:35:28.728Z{"msg":"Set desired generation to :186","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.596329568Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244202023-09-22T23:35:28.730Z{"msg":"[2] 8e3bcd14-e0d7-4242-b633-40317286a464 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.596458177Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927,"looper":"2"}
244212023-09-22T23:35:28.733Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.596516092Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244222023-09-22T23:35:28.736Z{"msg":"[2] 8e3bcd14-e0d7-4242-b633-40317286a464 (0d67e8d6-0ab5-4dca-8df5-6822e0f76e66) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.596922365Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244232023-09-22T23:35:28.739Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.596981457Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244242023-09-22T23:35:28.742Z{"msg":"[0] 8e3bcd14-e0d7-4242-b633-40317286a464 (0d67e8d6-0ab5-4dca-8df5-6822e0f76e66) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.597042919Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244252023-09-22T23:35:28.745Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.597100659Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244262023-09-22T23:35:28.748Z{"msg":"[2] client is_active_req TRUE, promote! session 0d67e8d6-0ab5-4dca-8df5-6822e0f76e66","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.597158526Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244272023-09-22T23:35:28.751Z{"msg":"[0] client is_active_req TRUE, promote! session 0d67e8d6-0ab5-4dca-8df5-6822e0f76e66","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.59722791Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244282023-09-22T23:35:28.753Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.597768945Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244292023-09-22T23:35:28.756Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.597824529Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244302023-09-22T23:35:28.759Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.597851587Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244312023-09-22T23:35:28.761Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.597897008Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244322023-09-22T23:35:28.764Z{"msg":"8e3bcd14-e0d7-4242-b633-40317286a464 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.598034872Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244332023-09-22T23:35:28.767Z{"msg":"8e3bcd14-e0d7-4242-b633-40317286a464 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.598155896Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244342023-09-22T23:35:28.769Z{"msg":"[0] 8e3bcd14-e0d7-4242-b633-40317286a464 (0d67e8d6-0ab5-4dca-8df5-6822e0f76e66) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.600254617Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244352023-09-22T23:35:28.772Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.600315732Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244362023-09-22T23:35:28.775Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:35:28.600360837Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244372023-09-22T23:35:28.778Z{"msg":"[2] 8e3bcd14-e0d7-4242-b633-40317286a464 (0d67e8d6-0ab5-4dca-8df5-6822e0f76e66) WaitQuorum New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.600384523Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244382023-09-22T23:35:28.780Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.600403052Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244392023-09-22T23:35:28.783Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:35:28.600439949Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244402023-09-22T23:35:28.786Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.600460168Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244412023-09-22T23:35:28.789Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.600496451Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244422023-09-22T23:35:28.792Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.600516742Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244432023-09-22T23:35:28.795Z{"msg":"8e3bcd14-e0d7-4242-b633-40317286a464 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.600534767Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244442023-09-22T23:35:28.798Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.600552445Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244452023-09-22T23:35:28.801Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.600569659Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244462023-09-22T23:35:28.803Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.600587612Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244472023-09-22T23:35:28.806Z{"msg":"8e3bcd14-e0d7-4242-b633-40317286a464 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.600605599Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244482023-09-22T23:35:28.809Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.600623042Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244492023-09-22T23:35:28.812Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:28.600639765Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244502023-09-22T23:35:29.598Z{"msg":"[1] 8e3bcd14-e0d7-4242-b633-40317286a464 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.59633574Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927,"looper":"1"}
244512023-09-22T23:35:29.602Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.596422479Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244522023-09-22T23:35:29.604Z{"msg":"[1] 8e3bcd14-e0d7-4242-b633-40317286a464 (0d67e8d6-0ab5-4dca-8df5-6822e0f76e66) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.597470925Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244532023-09-22T23:35:29.607Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.597529338Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244542023-09-22T23:35:29.610Z{"msg":"[1] client is_active_req TRUE, promote! session 0d67e8d6-0ab5-4dca-8df5-6822e0f76e66","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.597553356Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244552023-09-22T23:35:29.613Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.598225821Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244562023-09-22T23:35:29.616Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.598283563Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244572023-09-22T23:35:29.619Z{"msg":"8e3bcd14-e0d7-4242-b633-40317286a464 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.598308944Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244582023-09-22T23:35:29.622Z{"msg":"[1] 8e3bcd14-e0d7-4242-b633-40317286a464 (0d67e8d6-0ab5-4dca-8df5-6822e0f76e66) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.600742816Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244592023-09-22T23:35:29.625Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.600820751Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244602023-09-22T23:35:29.628Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:35:29.600890475Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244612023-09-22T23:35:29.632Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.60093871Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244622023-09-22T23:35:29.635Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.600979162Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244632023-09-22T23:35:29.637Z{"msg":"8e3bcd14-e0d7-4242-b633-40317286a464 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.601019112Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244642023-09-22T23:35:29.640Z{"msg":"[0]R flush_numbers[0..12]: [308, 308, 289, 294, 291, 309, 305, 301, 304, 302, 290, 307]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.60105973Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244652023-09-22T23:35:29.643Z{"msg":"[0]R generation[0..12]: [184, 184, 178, 181, 180, 185, 184, 182, 184, 183, 179, 184]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.601119733Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244662023-09-22T23:35:29.646Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":The guest has finished waiting for activation with:186
244672023-09-22T23:35:29.649ZWait for a query_work_queue command to finish before sending IO
244682023-09-22T23:35:29.652Z0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.601332998Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244692023-09-22T23:35:29.655Z{"msg":"[1]R flush_numbers[0..12]: [308, 308, 289, 294, 291, 309, 305, 301, 304, 302, 290, 307]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.60140932Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244702023-09-22T23:35:29.657Z{"msg":"[1]R generation[0..12]: [184, 184, 178, 181, 180, 185, 184, 182, 184, 183, 179, 184]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.601465871Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244712023-09-22T23:35:29.660Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.60151517Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244722023-09-22T23:35:29.663Z{"msg":"[2]R flush_numbers[0..12]: [308, 308, 289, 294, 291, 309, 305, 301, 304, 302, 290, 307]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.601555445Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244732023-09-22T23:35:29.666Z{"msg":"[2]R generation[0..12]: [184, 184, 178, 181, 180, 185, 184, 182, 184, 183, 179, 184]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.601597201Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244742023-09-22T23:35:29.670Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.601643561Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244752023-09-22T23:35:29.673Z{"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.601682189Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244762023-09-22T23:35:29.676Z{"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.601727509Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244772023-09-22T23:35:29.679Z{"msg":"Next flush: 310","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.601769795Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244782023-09-22T23:35:29.682Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.601807989Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244792023-09-22T23:35:29.685Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.601846391Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244802023-09-22T23:35:29.688Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.601886576Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244812023-09-22T23:35:29.691Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.601924974Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244822023-09-22T23:35:29.694Z{"msg":"8e3bcd14-e0d7-4242-b633-40317286a464 is now active with session: 0d67e8d6-0ab5-4dca-8df5-6822e0f76e66","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.601965001Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244832023-09-22T23:35:29.697Z{"msg":"8e3bcd14-e0d7-4242-b633-40317286a464 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.602005279Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244842023-09-22T23:35:29.700Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.602045053Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244852023-09-22T23:35:29.703Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.602082788Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244862023-09-22T23:35:29.706Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
244872023-09-22T23:35:29.709Z,"time":"2023-09-22T23:35:29.602121504Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244882023-09-22T23:35:29.712Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.602160212Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244892023-09-22T23:35:29.715Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.602197708Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244902023-09-22T23:35:29.718Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.602234848Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244912023-09-22T23:35:29.720Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.602272372Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244922023-09-22T23:35:29.723Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.602309335Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244932023-09-22T23:35:29.726Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.602346531Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244942023-09-22T23:35:29.729Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.602383715Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244952023-09-22T23:35:29.732Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.602421285Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1927}
244962023-09-22T23:35:29.734ZLoading write count information from file "/tmp/test_repair_verify.data"
244972023-09-22T23:35:29.738ZRun Repair workload
244982023-09-22T23:35:29.740Z01/30 Write block 307 len 20480 data: 17 17 16 13 12
244992023-09-22T23:35:29.743Z02/30 Read block 581 len 4096
245002023-09-22T23:35:29.746Z03/30 Write block 185 len 16384 data: 13 15 15 12
245012023-09-22T23:35:29.749Z04/30 Write block 466 len 8192 data: 17 16
245022023-09-22T23:35:29.752Z05/30 Write block 192 len 24576 data: 13 13 14 15 14 12
245032023-09-22T23:35:29.755Z06/30 Read block 480 len 20480
245042023-09-22T23:35:29.758Z07/30 Flush
245052023-09-22T23:35:29.761Z08/30 Write block 142 len 24576 data: 14 14 9 11 11 11
245062023-09-22T23:35:29.764Z09/30 Write block 545 len 12288 data: 10 14 12
245072023-09-22T23:35:29.766Z10/30 Read block 77 len 16384
245082023-09-22T23:35:29.769Z11/30 Write block 6 len 8192 data: 9 10
245092023-09-22T23:35:29.772Z12/30 Read block 270 len 8192
245102023-09-22T23:35:29.775Z13/30 Write block 525 len 16384 data: 18 18 20 17
245112023-09-22T23:35:29.778Z14/30 Read block 538 len 12288
245122023-09-22T23:35:29.781Z15/30 Write block 400 len 8192 data: 18 15
245132023-09-22T23:35:29.784Z16/30 Read block 299 len 4096
245142023-09-22T23:35:29.787Z17/30 Flush
245152023-09-22T23:35:29.790Z18/30 Write block 3 len 4096 data: 8
245162023-09-22T23:35:29.793Z19/30 Read block 301 len 36864
245172023-09-22T23:35:29.796Z20/30 Write block 501 len 20480 data: 13 14 13 13 13
245182023-09-22T23:35:29.799Z21/30 Read block 60 len 4096
245192023-09-22T23:35:29.802Z22/30 Read block 229 len 20480
245202023-09-22T23:35:29.805Z23/30 Read block 295 len 40960
245212023-09-22T23:35:29.808Z24/30 Write block 248 len 8192 data: 11 11
245222023-09-22T23:35:29.811Z25/30 Write block 245 len 32768 data: 12 8 11 12 12 11 10 14
245232023-09-22T23:35:29.813Z26/30 Write block 330 len 24576 data: 11 14 15 14 19 19
245242023-09-22T23:35:29.816Z27/30 Read block 530 len 40960
245252023-09-22T23:35:29.819Z28/30 Write block 478 len 16384 data: 13 14 14 14
245262023-09-22T23:35:29.822Z29/30 Write block 314 len 28672 data: 15 11 11 11 11 11 14
245272023-09-22T23:35:29.825Z30/30 Read block 415 len 28672
245282023-09-22T23:35:29.828Z----------------------------------------------------------------
245292023-09-22T23:35:29.831Z Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:23
245302023-09-22T23:35:29.834ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
245312023-09-22T23:35:29.837Z 8 Acked 1007 Write 6 Done Done Done false
245322023-09-22T23:35:29.840Z 9 Acked 1008 Write 3 Done Done Done false
245332023-09-22T23:35:29.843Z 10 Acked 1009 Read 4 Done Done Done false
245342023-09-22T23:35:29.846Z 11 Acked 1010 Write 2 Done Done Done false
245352023-09-22T23:35:29.849Z 12 Acked 1011 Read 2 Done Done Done false
245362023-09-22T23:35:29.852Z 13 Acked 1012 Write 4 Done Done Done false
245372023-09-22T23:35:29.855Z 14 Acked 1013 Read 3 Done Sent Done false
245382023-09-22T23:35:29.857Z 15 Acked 1014 Write 2 Done Sent Done false
245392023-09-22T23:35:29.860Z 16 Acked 1015 Read 1 Done Sent Done false
245402023-09-22T23:35:29.863Z 17 Acked 1016 Flush 0 Done Sent Done false
245412023-09-22T23:35:29.866Z 18 Acked 1017 Write 1 Done Sent Done false
245422023-09-22T23:35:29.869Z 19 Acked 1018 Read 9 Done Sent Done false
245432023-09-22T23:35:29.871Z 20 Acked 1019 Write 5 Done Sent Done false
245442023-09-22T23:35:29.874Z 21 Acked 1020 Read 1 Done Sent Done false
245452023-09-22T23:35:29.877Z 22 Acked 1021 Read 5 Done Sent Done false
245462023-09-22T23:35:29.880Z 23 Acked 1022 Read 10 Done Sent Done false
245472023-09-22T23:35:29.882Z 24 Acked 1023 Write 2 Done Sent Done false
245482023-09-22T23:35:29.885Z 25 Acked 1024 Write 8 Done Sent Done false
245492023-09-22T23:35:29.888Z 26 Acked 1025 Write 6 Done Sent Done false
245502023-09-22T23:35:29.891Z 27 Acked 1026 Read 10 Done Sent Done false
245512023-09-22T23:35:29.894Z 28 Acked 1027 Write 4 Done Sent Done false
245522023-09-22T23:35:29.896Z 29 Acked 1028 Write 7 Done Sent Done false
245532023-09-22T23:35:29.899Z 30 Acked 1029 Read 7 Sent Sent Done false
245542023-09-22T23:35:29.902Z STATES DS:0 DS:1 DS:2 TOTAL
245552023-09-22T23:35:29.905Z New 0 0 0 0
245562023-09-22T23:35:29.908Z Sent 1 17 0 18
245572023-09-22T23:35:29.911Z Done 22 6 23 51
245582023-09-22T23:35:29.914Z Skipped 0 0 0 0
245592023-09-22T23:35:29.916Z Error 0 0 0 0
245602023-09-22T23:35:29.919ZLast Flush: 1016 1006 1016
245612023-09-22T23:35:29.922ZDownstairs last five completed: 1006 1005 1004 1003 1002 1001
245622023-09-22T23:35:29.925ZUpstairs last five completed: 30 29 28 27 26 25
245632023-09-22T23:35:29.927ZWrote out file "/tmp/test_repair_verify.data"
245642023-09-22T23:35:29.930Z
245652023-09-22T23:35:29.933ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
245662023-09-22T23:35:29.936Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.730886732Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1928}
245672023-09-22T23:35:29.939Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.731246507Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1928}
245682023-09-22T23:35:29.941Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.731269813Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1928}
245692023-09-22T23:35:29.944Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.760959008Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1928}
245702023-09-22T23:35:29.947Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.761012763Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1928}
245712023-09-22T23:35:29.950Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.761051718Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1928}
245722023-09-22T23:35:29.953Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.797773462Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1928}
245732023-09-22T23:35:29.955Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.797833378Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1928}
245742023-09-22T23:35:29.958Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:29.79785645Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1928}
245752023-09-22T23:35:29.961ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
245762023-09-22T23:35:29.964Z 0 000-019 186 184 186 311 308 311 T T T <---
245772023-09-22T23:35:29.966Z 1 020-039 184 184 184 308 308 308 F F F
245782023-09-22T23:35:29.969Z 2 040-059 178 178 178 289 289 289 F F F
245792023-09-22T23:35:29.971Z 3 060-079 181 181 181 294 294 294 F F F
245802023-09-22T23:35:29.974Z 4 080-099 180 180 180 291 291 291 F F F
245812023-09-22T23:35:29.977Z 5 100-119 185 185 185 309 309 309 F F F
245822023-09-22T23:35:29.980Z 6 120-139 184 184 184 305 305 305 F F F
245832023-09-22T23:35:29.983Z 7 140-159 186 182 186 311 301 311 F T F <---
245842023-09-22T23:35:29.985Z 8 160-179 184 184 184 304 304 304 F F F
245852023-09-22T23:35:29.988Z 9 180-199 186 186 186 310 310 310 F F F
245862023-09-22T23:35:29.991Z 10 200-219 179 179 179 290 290 290 F F F
245872023-09-22T23:35:29.994Z 11 220-239 184 184 184 307 307 307 F F F
245882023-09-22T23:35:29.997Z 12 240-259 184 184 184 307 307 307 T F T <---
245892023-09-22T23:35:30.000Z 13 260-279 183 183 183 302 302 302 F F F
245902023-09-22T23:35:30.003Z 14 280-299 184 184 184 305 305 305 F F F
245912023-09-22T23:35:30.006ZError: Difference in extent metadata found!
245922023-09-22T23:35:30.009Z 15 300-319 186 186 186 310 310 310 T F T <---
245932023-09-22T23:35:30.012Z 16 320-339 184 184 184 305 305 305 T F T <---
245942023-09-22T23:35:30.014Z 17 340-359 184 184 184 305 305 305 F F F
245952023-09-22T23:35:30.017Z 18 360-379 182 182 182 300 300 300 F F F
245962023-09-22T23:35:30.020Z 19 380-399 182 182 182 297 297 297 F F F
245972023-09-22T23:35:30.023Z 20 400-419 186 184 186 311 306 311 F F F <---
245982023-09-22T23:35:30.026Z 21 420-439 184 184 184 306 306 306 F F F
245992023-09-22T23:35:30.029Z 22 440-459 183 183 183 302 302 302 F F F
246002023-09-22T23:35:30.032Z 23 460-479 186 186 186 310 310 310 T F T <---
246012023-09-22T23:35:30.034Z 24 480-499 183 183 183 302 302 302 T F T <---
246022023-09-22T23:35:30.037Z 25 500-519 182 182 182 299 299 299 T F T <---
246032023-09-22T23:35:30.040Z 26 520-539 186 184 186 311 307 311 F T F <---
246042023-09-22T23:35:30.043Z 27 540-559 186 182 186 311 298 311 F T F <---
246052023-09-22T23:35:30.046Z 28 560-579 179 179 179 290 290 290 F F F
246062023-09-22T23:35:30.049Z 29 580-599 183 183 183 302 302 302 F F F
246072023-09-22T23:35:30.051ZMax gen: 186, Max flush: 311
246082023-09-22T23:35:30.054ZA Difference in extent metadata is expected here
246092023-09-22T23:35:30.057ZOn loop 93
246102023-09-22T23:35:30.060Z
246112023-09-22T23:35:30.063ZVerifying data now
246122023-09-22T23:35:36.068ZNo change after verify
246132023-09-22T23:35:36.071ZLoop: 93 Downstairs dump after verify (and repair):
246142023-09-22T23:35:36.086Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.084615614Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1933}
246152023-09-22T23:35:36.089Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.085003398Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1933}
246162023-09-22T23:35:36.097Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.085044499Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1933}
246172023-09-22T23:35:36.105Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.103485456Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1933}
246182023-09-22T23:35:36.108Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.103542089Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1933}
246192023-09-22T23:35:36.113Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.103563309Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1933}
246202023-09-22T23:35:36.116Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.114404509Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1933}
246212023-09-22T23:35:36.119Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.114461429Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1933}
246222023-09-22T23:35:36.121Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.114482899Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1933}
246232023-09-22T23:35:36.126ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
246242023-09-22T23:35:36.129Z 0 000-019 187 187 187 312 312 312 F F F
246252023-09-22T23:35:36.132Z 1 020-039 184 184 184 308 308 308 F F F
246262023-09-22T23:35:36.135Z 2 040-059 178 178 178 289 289 289 F F F
246272023-09-22T23:35:36.138Z 3 060-079 181 181 181 294 294 294 F F F
246282023-09-22T23:35:36.141Z 4 080-099 180 180 180 291 291 291 F F F
246292023-09-22T23:35:36.144Z 5 100-119 185 185 185 309 309 309 F F F
246302023-09-22T23:35:36.146Z 6 120-139 184 184 184 305 305 305 F F F
246312023-09-22T23:35:36.149Z 7 140-159 186 186 186 311 311 311 F F F
246322023-09-22T23:35:36.152Z 8 160-179 184 184 184 304 304 304 F F F
246332023-09-22T23:35:36.155Z 9 180-199 186 186 186 310 310 310 F F F
246342023-09-22T23:35:36.158Z 10 200-219 179 179 179 290 290 290 F F F
246352023-09-22T23:35:36.161Z 11 220-239 184 184 184 307 307 307 F F F
246362023-09-22T23:35:36.164Z 12 240-259 187 187 187 312 312 312 F F F
246372023-09-22T23:35:36.167Z 13 260-279 183 183 183 302 302 302 F F F
246382023-09-22T23:35:36.170Z 14 280-299 184 184 184 305 305 305 F F F
246392023-09-22T23:35:36.173Z 15 300-319 187 187 187 312 312 312 F F F
246402023-09-22T23:35:36.176Z 16 320-339 187 187 187 312 312 312 F F F
246412023-09-22T23:35:36.178Z 17 340-359 184 184 184 305 305 305 F F F
246422023-09-22T23:35:36.181Z 18 360-379 182 182 182 300 300 300 F F F
246432023-09-22T23:35:36.184Z 19 380-399 182 182 182 297 297 297 F F F
246442023-09-22T23:35:36.187Z 20 400-419 186 186 186 311 311 311 F F F
246452023-09-22T23:35:36.190Z 21 420-439 184 184 184 306 306 306 F F F
246462023-09-22T23:35:36.192Z 22 440-459 183 183 183 302 302 302 F F F
246472023-09-22T23:35:36.195Z 23 460-479 187 187 187 312 312 312 F F F
246482023-09-22T23:35:36.198Z 24 480-499 187 187 187 312 312 312 F F F
246492023-09-22T23:35:36.201Z 25 500-519 187 187 187 312 312 312 F F F
246502023-09-22T23:35:36.203Z 26 520-539 186 186 186 311 311 311 F F F
246512023-09-22T23:35:36.206Z 27 540-559 186 186 186 311 311 311 F F F
246522023-09-22T23:35:36.210Z 28 560-579 179 179 179 290 290 290 F F F
246532023-09-22T23:35:36.213Z 29 580-599 183 183 183 302 302 302 F F F
246542023-09-22T23:35:36.216ZMax gen: 187, Max flush: 312
246552023-09-22T23:35:36.219Z
246562023-09-22T23:35:36.222ZBegin loop 94
246572023-09-22T23:35:36.224ZDownstairs to restart: 1
246582023-09-22T23:35:36.227Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.162586291Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246592023-09-22T23:35:36.230Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.163107825Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246602023-09-22T23:35:36.233Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.163188661Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246612023-09-22T23:35:36.236Z{"msg":"Crucible stats registered with UUID: 97fdd8a0-0255-47bb-828e-bbd59b9bd0bd","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.163262971Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246622023-09-22T23:35:36.239Z{"msg":"Crucible 97fdd8a0-0255-47bb-828e-bbd59b9bd0bd has session id: cde74289-3918-4b63-b71c-3985ce718739","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.163306906Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246632023-09-22T23:35:36.242ZCrucible runtime is spawned
246642023-09-22T23:35:36.245ZThe guest has requested activation with gen:188
246652023-09-22T23:35:36.249Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.163657901Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935,"task":"up_listen"}
246662023-09-22T23:35:36.253Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.163745455Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246672023-09-22T23:35:36.256Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.163826732Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246682023-09-22T23:35:36.259Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.163908579Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935,"looper":"1"}
246692023-09-22T23:35:36.261Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.163995771Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935,"looper":"2"}
246702023-09-22T23:35:36.265Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.164068628Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935,"looper":"0"}
246712023-09-22T23:35:36.268Z{"msg":"[2] 97fdd8a0-0255-47bb-828e-bbd59b9bd0bd looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.164151634Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935,"looper":"2"}
246722023-09-22T23:35:36.271Z{"msg":"[0] 97fdd8a0-0255-47bb-828e-bbd59b9bd0bd looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.164239857Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935,"looper":"0"}
246732023-09-22T23:35:36.275Z{"msg":"97fdd8a0-0255-47bb-828e-bbd59b9bd0bd active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.164314649Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246742023-09-22T23:35:36.278Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.1643797Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246752023-09-22T23:35:36.281Z{"msg":"Set desired generation to :188","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.164428547Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246762023-09-22T23:35:36.283Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.164509908Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246772023-09-22T23:35:36.286Z{"msg":"[2] 97fdd8a0-0255-47bb-828e-bbd59b9bd0bd (058df6a5-177c-4b45-af8f-20bb1fb45876) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.165320921Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246782023-09-22T23:35:36.289Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.165381942Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246792023-09-22T23:35:36.292Z{"msg":"[2] client is_active_req TRUE, promote! session 058df6a5-177c-4b45-af8f-20bb1fb45876","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.16542787Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246802023-09-22T23:35:36.295Z{"msg":"[0] 97fdd8a0-0255-47bb-828e-bbd59b9bd0bd (058df6a5-177c-4b45-af8f-20bb1fb45876) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.165472227Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246812023-09-22T23:35:36.298Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.165536201Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246822023-09-22T23:35:36.301Z{"msg":"[0] client is_active_req TRUE, promote! session 058df6a5-177c-4b45-af8f-20bb1fb45876","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.165610951Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246832023-09-22T23:35:36.303Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.166057398Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246842023-09-22T23:35:36.306Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.166123166Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246852023-09-22T23:35:36.309Z{"msg":"97fdd8a0-0255-47bb-828e-bbd59b9bd0bd WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.166245376Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246862023-09-22T23:35:36.312Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.166360079Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246872023-09-22T23:35:36.315Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.166470538Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246882023-09-22T23:35:36.317Z{"msg":"97fdd8a0-0255-47bb-828e-bbd59b9bd0bd WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.166545884Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246892023-09-22T23:35:36.320Z{"msg":"[2] 97fdd8a0-0255-47bb-828e-bbd59b9bd0bd (058df6a5-177c-4b45-af8f-20bb1fb45876) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.168290656Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246902023-09-22T23:35:36.324Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.168373244Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246912023-09-22T23:35:36.327Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:35:36.168448595Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246922023-09-22T23:35:36.330Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.168547952Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246932023-09-22T23:35:36.333Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.168653757Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246942023-09-22T23:35:36.336Z{"msg":"97fdd8a0-0255-47bb-828e-bbd59b9bd0bd WaitActive New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.168728051Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246952023-09-22T23:35:36.339Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.168800621Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246962023-09-22T23:35:36.342Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.168863061Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246972023-09-22T23:35:36.345Z{"msg":"[0] 97fdd8a0-0255-47bb-828e-bbd59b9bd0bd (058df6a5-177c-4b45-af8f-20bb1fb45876) WaitActive New WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.168909411Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246982023-09-22T23:35:36.347Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.16894852Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
246992023-09-22T23:35:36.350Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:35:36.169005481Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247002023-09-22T23:35:36.353Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.169052241Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247012023-09-22T23:35:36.356Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.169098118Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247022023-09-22T23:35:36.358Z{"msg":"97fdd8a0-0255-47bb-828e-bbd59b9bd0bd WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.169226334Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247032023-09-22T23:35:36.361Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.169287355Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247042023-09-22T23:35:36.364Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:36.169330682Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247052023-09-22T23:35:37.166Z{"msg":"[1] 97fdd8a0-0255-47bb-828e-bbd59b9bd0bd looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.164679858Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935,"looper":"1"}
247062023-09-22T23:35:37.169Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.164765445Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247072023-09-22T23:35:37.172Z{"msg":"[1] 97fdd8a0-0255-47bb-828e-bbd59b9bd0bd (058df6a5-177c-4b45-af8f-20bb1fb45876) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.165922752Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247082023-09-22T23:35:37.176Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.165979659Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247092023-09-22T23:35:37.179Z{"msg":"[1] client is_active_req TRUE, promote! session 058df6a5-177c-4b45-af8f-20bb1fb45876","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.166024349Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247102023-09-22T23:35:37.182Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.16659282Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247112023-09-22T23:35:37.185Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.166650333Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247122023-09-22T23:35:37.188Z{"msg":"97fdd8a0-0255-47bb-828e-bbd59b9bd0bd WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.16669519Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247132023-09-22T23:35:37.191Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.169099371Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247142023-09-22T23:35:37.194Z{"msg":"[1] 97fdd8a0-0255-47bb-828e-bbd59b9bd0bd (058df6a5-177c-4b45-af8f-20bb1fb45876) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.16923951Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247152023-09-22T23:35:37.197Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.16929218Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247162023-09-22T23:35:37.200Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:35:37.169332904Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247172023-09-22T23:35:37.203Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.169372286Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247182023-09-22T23:35:37.206Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.169411689Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247192023-09-22T23:35:37.209Z{"msg":"97fdd8a0-0255-47bb-828e-bbd59b9bd0bd WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.169451703Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247202023-09-22T23:35:37.212Z{"msg":"[0]R flush_numbers[0..12]: [312, 308, 289, 294, 291, 309, 305, 311, 304, 310, 290, 307]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.169492123Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247212023-09-22T23:35:37.215Z{"msg":"[0]R generation[0..12]: [187, 184, 178, 181, 180, 185, 184, 186, 184, 186, 179, 184]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.169514252Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247222023-09-22T23:35:37.218Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.16953393Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247232023-09-22T23:35:37.222Z{"msg":"[1]R flush_numbers[0..12]: [312, 308, 289, 294, 291, 309, 305, 311, 304, 310, 290, 307]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.169572324Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247242023-09-22T23:35:37.225ZThe guest has finished waiting for activation with:188
247252023-09-22T23:35:37.229Z{"msg":"[1]R generation[0..12]: [187, 184, 178, 181, 180, 185, 184, 186, 184, 186, 179, 184]"Wait for a query_work_queue command to finish before sending IO
247262023-09-22T23:35:37.231Z,"v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.169624863Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247272023-09-22T23:35:37.235Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.169665634Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247282023-09-22T23:35:37.238Z{"msg":"[2]R flush_numbers[0..12]: [312, 308, 289, 294, 291, 309, 305, 311, 304, 310, 290, 307]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.16970527Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247292023-09-22T23:35:37.240Z{"msg":"[2]R generation[0..12]: [187, 184, 178, 181, 180, 185, 184, 186, 184, 186, 179, 184]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.169752409Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247302023-09-22T23:35:37.243Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.169804786Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247312023-09-22T23:35:37.247Z{"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.16986764Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247322023-09-22T23:35:37.250Z{"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.16993881Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247332023-09-22T23:35:37.253Z{"msg":"Next flush: 313","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.170003459Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247342023-09-22T23:35:37.256Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.170056988Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247352023-09-22T23:35:37.259Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.170098482Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247362023-09-22T23:35:37.262Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.17013833Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247372023-09-22T23:35:37.265Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.170177529Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247382023-09-22T23:35:37.268Z{"msg":"97fdd8a0-0255-47bb-828e-bbd59b9bd0bd is now active with session: 058df6a5-177c-4b45-af8f-20bb1fb45876","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.170223638Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247392023-09-22T23:35:37.271Z{"msg":"97fdd8a0-0255-47bb-828e-bbd59b9bd0bd Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.17027115Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247402023-09-22T23:35:37.274Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.170309968Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247412023-09-22T23:35:37.278Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.17034865Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247422023-09-22T23:35:37.281Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.170395704Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247432023-09-22T23:35:37.284Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.170450405Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247442023-09-22T23:35:37.287Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
247452023-09-22T23:35:37.290Z,"time":"2023-09-22T23:35:37.170505367Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247462023-09-22T23:35:37.293Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.170615682Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247472023-09-22T23:35:37.296Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.170661351Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247482023-09-22T23:35:37.299Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.17070044Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247492023-09-22T23:35:37.302Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.170740128Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247502023-09-22T23:35:37.306Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.170779106Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247512023-09-22T23:35:37.310Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.170817201Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1935}
247522023-09-22T23:35:37.313ZLoading write count information from file "/tmp/test_repair_verify.data"
247532023-09-22T23:35:37.316ZRun Repair workload
247542023-09-22T23:35:37.319Z01/30 Write block 542 len 28672 data: 11 12 9 11 15 13 15
247552023-09-22T23:35:37.322Z02/30 Read block 530 len 36864
247562023-09-22T23:35:37.325Z03/30 Write block 598 len 4096 data: 6
247572023-09-22T23:35:37.328Z04/30 Write block 257 len 24576 data: 9 11 11 12 16 16
247582023-09-22T23:35:37.331Z05/30 Read block 572 len 36864
247592023-09-22T23:35:37.335Z06/30 Write block 364 len 36864 data: 9 9 10 11 14 13 10 12 12
247602023-09-22T23:35:37.338Z07/30 Flush
247612023-09-22T23:35:37.341Z08/30 Write block 177 len 28672 data: 20 18 15 12 14 15 12
247622023-09-22T23:35:37.344Z09/30 Flush
247632023-09-22T23:35:37.347Z10/30 Write block 238 len 12288 data: 13 15 18
247642023-09-22T23:35:37.351Z11/30 Read block 209 len 36864
247652023-09-22T23:35:37.353Z12/30 Read block 464 len 32768
247662023-09-22T23:35:37.356Z13/30 Read block 21 len 8192
247672023-09-22T23:35:37.359Z14/30 Read block 376 len 28672
247682023-09-22T23:35:37.362Z15/30 Write block 228 len 8192 data: 10 9
247692023-09-22T23:35:37.365Z16/30 Read block 154 len 4096
247702023-09-22T23:35:37.368Z17/30 Read block 251 len 16384
247712023-09-22T23:35:37.371Z18/30 Write block 54 len 28672 data: 7 8 9 12 14 15 18
247722023-09-22T23:35:37.374Z19/30 Write block 456 len 32768 data: 13 13 14 17 17 16 15 15
247732023-09-22T23:35:37.377Z20/30 Read block 108 len 28672
247742023-09-22T23:35:37.379Z21/30 Read block 394 len 8192
247752023-09-22T23:35:37.382Z22/30 Write block 385 len 8192 data: 13 11
247762023-09-22T23:35:37.385Z23/30 Read block 482 len 32768
247772023-09-22T23:35:37.388Z24/30 Flush
247782023-09-22T23:35:37.391Z25/30 Write block 207 len 8192 data: 21 25
247792023-09-22T23:35:37.394Z26/30 Read block 242 len 28672
247802023-09-22T23:35:37.396Z27/30 Read block 375 len 20480
247812023-09-22T23:35:37.403Z28/30 Read block 220 len 24576
247822023-09-22T23:35:37.406Z29/30 Write block 193 len 36864 data: 14 15 16 15 13 9 8 10 13
247832023-09-22T23:35:37.409Z30/30 Write block 374 len 24576 data: 11 10 10 12 14 13
247842023-09-22T23:35:37.412Z----------------------------------------------------------------
247852023-09-22T23:35:37.415Z Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30
247862023-09-22T23:35:37.418ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
247872023-09-22T23:35:37.421Z 1 Acked 1000 Write 7 Done Sent Done false
247882023-09-22T23:35:37.424Z 2 Acked 1001 Read 9 Done Sent Done false
247892023-09-22T23:35:37.427Z 3 Acked 1002 Write 1 Done Sent Done false
247902023-09-22T23:35:37.430Z 4 Acked 1003 Write 6 Done Sent Done false
247912023-09-22T23:35:37.433Z 5 Acked 1004 Read 9 Done Sent Done false
247922023-09-22T23:35:37.436Z 6 Acked 1005 Write 9 Done Sent Done false
247932023-09-22T23:35:37.439Z 7 Acked 1006 Flush 0 Done Sent Done false
247942023-09-22T23:35:37.441Z 8 Acked 1007 Write 7 Done Sent Done false
247952023-09-22T23:35:37.444Z 9 Acked 1008 Flush 0 Done Sent Done false
247962023-09-22T23:35:37.447Z 10 Acked 1009 Write 3 Done Sent Done false
247972023-09-22T23:35:37.450Z 11 Acked 1010 Read 9 Done Sent Done false
247982023-09-22T23:35:37.452Z 12 Acked 1011 Read 8 Done Sent Done false
247992023-09-22T23:35:37.455Z 13 Acked 1012 Read 2 Done Sent Done false
248002023-09-22T23:35:37.458Z 14 Acked 1013 Read 7 Done Sent Done false
248012023-09-22T23:35:37.461Z 15 Acked 1014 Write 2 Done Sent Done false
248022023-09-22T23:35:37.464Z 16 Acked 1015 Read 1 Done Sent Done false
248032023-09-22T23:35:37.467Z 17 Acked 1016 Read 4 Done Sent Done false
248042023-09-22T23:35:37.470Z 18 Acked 1017 Write 7 Done Sent Done false
248052023-09-22T23:35:37.473Z 19 Acked 1018 Write 8 Done Sent Done false
248062023-09-22T23:35:37.478Z 20 Acked 1019 Read 7 Done Sent Done false
248072023-09-22T23:35:37.481Z 21 Acked 1020 Read 2 Done Sent Done false
248082023-09-22T23:35:37.484Z 22 Acked 1021 Write 2 Done Sent Done false
248092023-09-22T23:35:37.488Z 23 Acked 1022 Read 8 Done Sent Done false
248102023-09-22T23:35:37.491Z 24 Acked 1023 Flush 0 Done Sent Done false
248112023-09-22T23:35:37.493Z 25 Acked 1024 Write 2 Done Sent Done false
248122023-09-22T23:35:37.496Z 26 Acked 1025 Read 7 Done Sent Done false
248132023-09-22T23:35:37.499Z 27 Acked 1026 Read 5 Done Sent Done false
248142023-09-22T23:35:37.502Z 28 Acked 1027 Read 6 Done Sent Done false
248152023-09-22T23:35:37.505Z 29 Acked 1028 Write 9 Sent Sent Sent false
248162023-09-22T23:35:37.508Z 30 Acked 1029 Write 6 Sent Sent Sent false
248172023-09-22T23:35:37.510Z STATES DS:0 DS:1 DS:2 TOTAL
248182023-09-22T23:35:37.513Z New 0 0 0 0
248192023-09-22T23:35:37.516Z Sent 2 30 2 34
248202023-09-22T23:35:37.519Z Done 28 0 28 56
248212023-09-22T23:35:37.522Z Skipped 0 0 0 0
248222023-09-22T23:35:37.525Z Error 0 0 0 0
248232023-09-22T23:35:37.528ZLast Flush: 1023 0 1023
248242023-09-22T23:35:37.531ZDownstairs last five completed:
248252023-09-22T23:35:37.533ZUpstairs last five completed: 30 29 28 27 26 25
248262023-09-22T23:35:37.536ZWrote out file "/tmp/test_repair_verify.data"
248272023-09-22T23:35:37.539Z
248282023-09-22T23:35:37.542ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
248292023-09-22T23:35:37.544Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.312098619Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1936}
248302023-09-22T23:35:37.547Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.312383898Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1936}
248312023-09-22T23:35:37.551Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.31240711Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1936}
248322023-09-22T23:35:37.554Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.336732081Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1936}
248332023-09-22T23:35:37.556Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.336776364Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1936}
248342023-09-22T23:35:37.559Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.336797407Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1936}
248352023-09-22T23:35:37.562Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.352075715Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1936}
248362023-09-22T23:35:37.565ZError: Difference in extent metadata found!
248372023-09-22T23:35:37.568Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.352106699Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1936}
248382023-09-22T23:35:37.570Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:37.352125768Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1936}
248392023-09-22T23:35:37.573ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
248402023-09-22T23:35:37.576Z 0 000-019 187 187 187 312 312 312 F F F
248412023-09-22T23:35:37.579Z 1 020-039 184 184 184 308 308 308 F F F
248422023-09-22T23:35:37.582Z 2 040-059 188 178 188 315 289 315 F F F <---
248432023-09-22T23:35:37.585Z 3 060-079 188 181 188 315 294 315 F F F <---
248442023-09-22T23:35:37.588Z 4 080-099 180 180 180 291 291 291 F F F
248452023-09-22T23:35:37.590Z 5 100-119 185 185 185 309 309 309 F F F
248462023-09-22T23:35:37.593Z 6 120-139 184 184 184 305 305 305 F F F
248472023-09-22T23:35:37.596Z 7 140-159 186 186 186 311 311 311 F F F
248482023-09-22T23:35:37.599Z 8 160-179 188 184 188 314 304 314 F F F <---
248492023-09-22T23:35:37.603Z 9 180-199 188 186 188 314 310 314 T F T <---
248502023-09-22T23:35:37.606Z 10 200-219 179 179 179 290 290 290 T F T <---
248512023-09-22T23:35:37.609Z 11 220-239 188 184 188 315 307 315 F F F <---
248522023-09-22T23:35:37.612Z 12 240-259 188 187 188 315 312 315 F F F <---
248532023-09-22T23:35:37.615Z 13 260-279 188 183 188 313 302 313 F F F <---
248542023-09-22T23:35:37.617Z 14 280-299 184 184 184 305 305 305 F F F
248552023-09-22T23:35:37.620Z 15 300-319 187 187 187 312 312 312 F F F
248562023-09-22T23:35:37.623Z 16 320-339 187 187 187 312 312 312 F F F
248572023-09-22T23:35:37.626Z 17 340-359 184 184 184 305 305 305 F F F
248582023-09-22T23:35:37.628Z 18 360-379 188 182 188 313 300 313 T F T <---
248592023-09-22T23:35:37.631Z 19 380-399 188 182 188 315 297 315 F F F <---
248602023-09-22T23:35:37.634Z 20 400-419 186 186 186 311 311 311 F F F
248612023-09-22T23:35:37.636Z 21 420-439 184 184 184 306 306 306 F F F
248622023-09-22T23:35:37.639Z 22 440-459 188 183 188 315 302 315 F F F <---
248632023-09-22T23:35:37.642Z 23 460-479 188 187 188 315 312 315 F F F <---
248642023-09-22T23:35:37.645Z 24 480-499 187 187 187 312 312 312 F F F
248652023-09-22T23:35:37.648Z 25 500-519 187 187 187 312 312 312 F F F
248662023-09-22T23:35:37.650Z 26 520-539 186 186 186 311 311 311 F F F
248672023-09-22T23:35:37.653Z 27 540-559 188 186 188 313 311 313 F F F <---
248682023-09-22T23:35:37.655Z 28 560-579 179 179 179 290 290 290 F F F
248692023-09-22T23:35:37.658Z 29 580-599 188 183 188 313 302 313 F F F <---
248702023-09-22T23:35:37.661ZMax gen: 188, Max flush: 315
248712023-09-22T23:35:37.664ZA Difference in extent metadata is expected here
248722023-09-22T23:35:37.667ZOn loop 94
248732023-09-22T23:35:37.669Z
248742023-09-22T23:35:37.672ZVerifying data now
248752023-09-22T23:35:43.850ZNo change after verify
248762023-09-22T23:35:43.853ZLoop: 94 Downstairs dump after verify (and repair):
248772023-09-22T23:35:43.868Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.866684897Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1941}
248782023-09-22T23:35:43.871Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.867003056Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1941}
248792023-09-22T23:35:43.874Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.867026719Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1941}
248802023-09-22T23:35:43.899Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.885474412Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1941}
248812023-09-22T23:35:43.902Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.88552661Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1941}
248822023-09-22T23:35:43.904Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.885546946Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1941}
248832023-09-22T23:35:43.907Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.896451585Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1941}
248842023-09-22T23:35:43.910Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.896502829Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1941}
248852023-09-22T23:35:43.912Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.896540617Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1941}
248862023-09-22T23:35:43.915ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
248872023-09-22T23:35:43.918Z 0 000-019 187 187 187 312 312 312 F F F
248882023-09-22T23:35:43.921Z 1 020-039 184 184 184 308 308 308 F F F
248892023-09-22T23:35:43.923Z 2 040-059 188 188 188 315 315 315 F F F
248902023-09-22T23:35:43.926Z 3 060-079 188 188 188 315 315 315 F F F
248912023-09-22T23:35:43.929Z 4 080-099 180 180 180 291 291 291 F F F
248922023-09-22T23:35:43.931Z 5 100-119 185 185 185 309 309 309 F F F
248932023-09-22T23:35:43.934Z 6 120-139 184 184 184 305 305 305 F F F
248942023-09-22T23:35:43.937Z 7 140-159 186 186 186 311 311 311 F F F
248952023-09-22T23:35:43.940Z 8 160-179 188 188 188 314 314 314 F F F
248962023-09-22T23:35:43.943Z 9 180-199 189 189 189 316 316 316 F F F
248972023-09-22T23:35:43.945Z 10 200-219 189 189 189 316 316 316 F F F
248982023-09-22T23:35:43.948Z 11 220-239 188 188 188 315 315 315 F F F
248992023-09-22T23:35:43.951Z 12 240-259 188 188 188 315 315 315 F F F
249002023-09-22T23:35:43.954Z 13 260-279 188 188 188 313 313 313 F F F
249012023-09-22T23:35:43.957Z 14 280-299 184 184 184 305 305 305 F F F
249022023-09-22T23:35:43.960Z 15 300-319 187 187 187 312 312 312 F F F
249032023-09-22T23:35:43.963Z 16 320-339 187 187 187 312 312 312 F F F
249042023-09-22T23:35:43.966Z 17 340-359 184 184 184 305 305 305 F F F
249052023-09-22T23:35:43.969Z 18 360-379 189 189 189 316 316 316 F F F
249062023-09-22T23:35:43.971Z 19 380-399 188 188 188 315 315 315 F F F
249072023-09-22T23:35:43.974Z 20 400-419 186 186 186 311 311 311 F F F
249082023-09-22T23:35:43.977Z 21 420-439 184 184 184 306 306 306 F F F
249092023-09-22T23:35:43.980Z 22 440-459 188 188 188 315 315 315 F F F
249102023-09-22T23:35:43.982Z 23 460-479 188 188 188 315 315 315 F F F
249112023-09-22T23:35:43.985Z 24 480-499 187 187 187 312 312 312 F F F
249122023-09-22T23:35:43.988Z 25 500-519 187 187 187 312 312 312 F F F
249132023-09-22T23:35:43.991Z 26 520-539 186 186 186 311 311 311 F F F
249142023-09-22T23:35:43.994Z 27 540-559 188 188 188 313 313 313 F F F
249152023-09-22T23:35:43.996Z 28 560-579 179 179 179 290 290 290 F F F
249162023-09-22T23:35:43.999Z 29 580-599 188 188 188 313 313 313 F F F
249172023-09-22T23:35:44.002ZMax gen: 189, Max flush: 316
249182023-09-22T23:35:44.005Z
249192023-09-22T23:35:44.008ZBegin loop 95
249202023-09-22T23:35:44.011ZDownstairs to restart: 0
249212023-09-22T23:35:44.014Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.944262781Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249222023-09-22T23:35:44.016Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.944758377Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249232023-09-22T23:35:44.020Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.944790588Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249242023-09-22T23:35:44.022Z{"msg":"Crucible stats registered with UUID: 07f3ac02-88ba-4c11-8f3e-158b0f830df4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.944847217Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249252023-09-22T23:35:44.025Z{"msg":"Crucible 07f3ac02-88ba-4c11-8f3e-158b0f830df4 has session id: cafb0464-9458-4f73-8df2-586ea9a93b5d","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.944870668Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249262023-09-22T23:35:44.028ZCrucible runtime is spawned
249272023-09-22T23:35:44.031Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.945175884Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943,"task":"up_listen"}
249282023-09-22T23:35:44.034Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.94522182Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249292023-09-22T23:35:44.037ZThe guest has requested activation with gen:190
249302023-09-22T23:35:44.039Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.9452826Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249312023-09-22T23:35:44.042Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.945340195Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943,"looper":"0"}
249322023-09-22T23:35:44.045Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.945624746Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943,"looper":"2"}
249332023-09-22T23:35:44.048Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.94581366Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943,"looper":"1"}
249342023-09-22T23:35:44.051Z{"msg":"[2] 07f3ac02-88ba-4c11-8f3e-158b0f830df4 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.945880371Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943,"looper":"2"}
249352023-09-22T23:35:44.054Z{"msg":"[1] 07f3ac02-88ba-4c11-8f3e-158b0f830df4 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.946017498Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943,"looper":"1"}
249362023-09-22T23:35:44.057Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.946101555Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249372023-09-22T23:35:44.060Z{"msg":"07f3ac02-88ba-4c11-8f3e-158b0f830df4 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.946168899Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249382023-09-22T23:35:44.062Z{"msg":"Set desired generation to :190","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.946229532Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249392023-09-22T23:35:44.065Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.946559898Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249402023-09-22T23:35:44.068Z{"msg":"[2] 07f3ac02-88ba-4c11-8f3e-158b0f830df4 (438aefcf-6edd-4542-ab81-07ef36f2e22c) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.946822874Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249412023-09-22T23:35:44.070Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.946974705Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249422023-09-22T23:35:44.073Z{"msg":"[2] client is_active_req TRUE, promote! session 438aefcf-6edd-4542-ab81-07ef36f2e22c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.947147839Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249432023-09-22T23:35:44.075Z{"msg":"[1] 07f3ac02-88ba-4c11-8f3e-158b0f830df4 (438aefcf-6edd-4542-ab81-07ef36f2e22c) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.94720826Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249442023-09-22T23:35:44.078Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.947310657Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249452023-09-22T23:35:44.081Z{"msg":"[1] client is_active_req TRUE, promote! session 438aefcf-6edd-4542-ab81-07ef36f2e22c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.947396835Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249462023-09-22T23:35:44.083Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.947683795Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249472023-09-22T23:35:44.086Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.94784519Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249482023-09-22T23:35:44.089Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.947922776Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249492023-09-22T23:35:44.092Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.948003834Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249502023-09-22T23:35:44.095Z{"msg":"07f3ac02-88ba-4c11-8f3e-158b0f830df4 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.948055425Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249512023-09-22T23:35:44.098Z{"msg":"07f3ac02-88ba-4c11-8f3e-158b0f830df4 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.948079745Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249522023-09-22T23:35:44.100Z{"msg":"[2] 07f3ac02-88ba-4c11-8f3e-158b0f830df4 (438aefcf-6edd-4542-ab81-07ef36f2e22c) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.950017447Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249532023-09-22T23:35:44.103Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.950094598Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249542023-09-22T23:35:44.106Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:35:43.950164214Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249552023-09-22T23:35:44.108Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.950235862Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249562023-09-22T23:35:44.112Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.950311384Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249572023-09-22T23:35:44.114Z{"msg":"07f3ac02-88ba-4c11-8f3e-158b0f830df4 New WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.950357247Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249582023-09-22T23:35:44.117Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.950422567Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249592023-09-22T23:35:44.120Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.950502546Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249602023-09-22T23:35:44.122Z{"msg":"[1] 07f3ac02-88ba-4c11-8f3e-158b0f830df4 (438aefcf-6edd-4542-ab81-07ef36f2e22c) New WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.950573862Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249612023-09-22T23:35:44.125Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.950619338Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249622023-09-22T23:35:44.128Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:35:43.950659085Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249632023-09-22T23:35:44.130Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.950697525Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249642023-09-22T23:35:44.133Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.950718205Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249652023-09-22T23:35:44.135Z{"msg":"07f3ac02-88ba-4c11-8f3e-158b0f830df4 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.950754967Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249662023-09-22T23:35:44.138Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.950774962Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249672023-09-22T23:35:44.141Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:43.950792212Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249682023-09-22T23:35:44.948Z{"msg":"[0] 07f3ac02-88ba-4c11-8f3e-158b0f830df4 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.946492488Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943,"looper":"0"}
249692023-09-22T23:35:44.952Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.946556218Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249702023-09-22T23:35:44.954Z{"msg":"[0] 07f3ac02-88ba-4c11-8f3e-158b0f830df4 (438aefcf-6edd-4542-ab81-07ef36f2e22c) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.947633201Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249712023-09-22T23:35:44.957Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.947700967Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249722023-09-22T23:35:44.960Z{"msg":"[0] client is_active_req TRUE, promote! session 438aefcf-6edd-4542-ab81-07ef36f2e22c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.947745694Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249732023-09-22T23:35:44.963Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.948387456Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249742023-09-22T23:35:44.965Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.948448608Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249752023-09-22T23:35:44.968Z{"msg":"07f3ac02-88ba-4c11-8f3e-158b0f830df4 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.948492624Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249762023-09-22T23:35:44.971Z{"msg":"[0] 07f3ac02-88ba-4c11-8f3e-158b0f830df4 (438aefcf-6edd-4542-ab81-07ef36f2e22c) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.950847684Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249772023-09-22T23:35:44.973Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.950942655Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249782023-09-22T23:35:44.976Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:35:44.950993043Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249792023-09-22T23:35:44.979Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.951033649Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249802023-09-22T23:35:44.983Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.951073472Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249812023-09-22T23:35:44.986Z{"msg":"07f3ac02-88ba-4c11-8f3e-158b0f830df4 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.951113913Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249822023-09-22T23:35:44.989Z{"msg":"[0]R flush_numbers[0..12]: [312, 308, 315, 315, 291, 309, 305, 311, 314, 316, 316, 315]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.951154414Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249832023-09-22T23:35:44.991Z{"msg":"[0]R generation[0..12]: [187, 184, 188, 188, 180, 185, 184, 186, 188, 189, 189, 188]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.951205326Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249842023-09-22T23:35:44.994Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.951246305Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":The guest has finished waiting for activation with:190
249852023-09-22T23:35:44.997Z1943}
249862023-09-22T23:35:45.000Z{Wait for a query_work_queue command to finish before sending IO
249872023-09-22T23:35:45.003Z"msg":"[1]R flush_numbers[0..12]: [312, 308, 315, 315, 291, 309, 305, 311, 314, 316, 316, 315]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.951276106Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249882023-09-22T23:35:45.006Z{"msg":"[1]R generation[0..12]: [187, 184, 188, 188, 180, 185, 184, 186, 188, 189, 189, 188]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.951310523Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249892023-09-22T23:35:45.009Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.95138399Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249902023-09-22T23:35:45.011Z{"msg":"[2]R flush_numbers[0..12]: [312, 308, 315, 315, 291, 309, 305, 311, 314, 316, 316, 315]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.951457225Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249912023-09-22T23:35:45.014Z{"msg":"[2]R generation[0..12]: [187, 184, 188, 188, 180, 185, 184, 186, 188, 189, 189, 188]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.951516307Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249922023-09-22T23:35:45.017Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.951569584Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249932023-09-22T23:35:45.020Z{"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.951591121Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249942023-09-22T23:35:45.022Z{"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.951632212Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249952023-09-22T23:35:45.025Z{"msg":"Next flush: 317","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.951652864Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249962023-09-22T23:35:45.028Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.951689233Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249972023-09-22T23:35:45.031Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.951727949Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249982023-09-22T23:35:45.034Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.951770975Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
249992023-09-22T23:35:45.037Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.951812425Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
250002023-09-22T23:35:45.040Z{"msg":"07f3ac02-88ba-4c11-8f3e-158b0f830df4 is now active with session: 438aefcf-6edd-4542-ab81-07ef36f2e22c","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.951852195Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
250012023-09-22T23:35:45.043Z{"msg":"07f3ac02-88ba-4c11-8f3e-158b0f830df4 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.951898575Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
250022023-09-22T23:35:45.045Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.951945695Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
250032023-09-22T23:35:45.048Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.951992786Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
250042023-09-22T23:35:45.051Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.952031288Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
250052023-09-22T23:35:45.054Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.952069116Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
250062023-09-22T23:35:45.059Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.952115429Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
250072023-09-22T23:35:45.063Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.95216337Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
250082023-09-22T23:35:45.066Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.952201832Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
250092023-09-22T23:35:45.069Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.95223971Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
250102023-09-22T23:35:45.072Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
250112023-09-22T23:35:45.074Z,"time":"2023-09-22T23:35:44.95227808Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
250122023-09-22T23:35:45.077Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.952383327Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
250132023-09-22T23:35:45.080Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:44.952428398Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1943}
250142023-09-22T23:35:45.082ZLoading write count information from file "/tmp/test_repair_verify.data"
250152023-09-22T23:35:45.085ZRun Repair workload
250162023-09-22T23:35:45.087Z01/30 Write block 263 len 20480 data: 18 17 15 15 18
250172023-09-22T23:35:45.090Z02/30 Read block 62 len 28672
250182023-09-22T23:35:45.093Z03/30 Write block 56 len 12288 data: 10 13 15
250192023-09-22T23:35:45.095Z04/30 Read block 550 len 36864
250202023-09-22T23:35:45.098Z05/30 Read block 377 len 28672
250212023-09-22T23:35:45.101Z06/30 Read block 356 len 32768
250222023-09-22T23:35:45.104Z07/30 Flush
250232023-09-22T23:35:45.106Z08/30 Write block 501 len 8192 data: 14 15
250242023-09-22T23:35:45.109Z09/30 Read block 182 len 32768
250252023-09-22T23:35:45.112Z10/30 Flush
250262023-09-22T23:35:45.114Z11/30 Write block 442 len 36864 data: 17 20 19 18 11 9 13 14 16
250272023-09-22T23:35:45.117Z12/30 Read block 378 len 8192
250282023-09-22T23:35:45.119Z13/30 Write block 320 len 24576 data: 15 13 10 13 11 8
250292023-09-22T23:35:45.122Z14/30 Write block 349 len 8192 data: 14 15
250302023-09-22T23:35:45.125Z15/30 Write block 559 len 40960 data: 13 15 13 13 13 15 15 15 17 17
250312023-09-22T23:35:45.128Z16/30 Write block 140 len 20480 data: 14 13 15 15 10
250322023-09-22T23:35:45.130Z17/30 Write block 491 len 36864 data: 25 21 20 18 20 20 21 14 14
250332023-09-22T23:35:45.133Z18/30 Read block 576 len 8192
250342023-09-22T23:35:45.136Z19/30 Read block 416 len 32768
250352023-09-22T23:35:45.139Z20/30 Write block 480 len 12288 data: 15 15 17
250362023-09-22T23:35:45.141Z21/30 Write block 459 len 40960 data: 18 18 17 16 16 14 18 18 17 18
250372023-09-22T23:35:45.145Z22/30 Write block 549 len 8192 data: 18 17
250382023-09-22T23:35:45.147Z23/30 Read block 380 len 12288
250392023-09-22T23:35:45.150Z24/30 Write block 419 len 8192 data: 18 16
250402023-09-22T23:35:45.153Z25/30 Write block 355 len 8192 data: 17 16
250412023-09-22T23:35:45.156Z26/30 Read block 0 len 32768
250422023-09-22T23:35:45.158Z27/30 Read block 19 len 28672
250432023-09-22T23:35:45.161Z28/30 Write block 15 len 12288 data: 14 13 13
250442023-09-22T23:35:45.163Z29/30 Write block 104 len 36864 data: 23 25 22 22 23 22 23 20 20
250452023-09-22T23:35:45.166Z30/30 Read block 219 len 32768
250462023-09-22T23:35:45.169Z----------------------------------------------------------------
250472023-09-22T23:35:45.172Z Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:30
250482023-09-22T23:35:45.175ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
250492023-09-22T23:35:45.177Z 1 Acked 1000 Write 5 Done Done Done false
250502023-09-22T23:35:45.180Z 2 Acked 1001 Read 7 Done Done Done false
250512023-09-22T23:35:45.183Z 3 Acked 1002 Write 3 Sent Done Done false
250522023-09-22T23:35:45.186Z 4 Acked 1003 Read 9 Sent Done Done false
250532023-09-22T23:35:45.189Z 5 Acked 1004 Read 7 Sent Done Done false
250542023-09-22T23:35:45.192Z 6 Acked 1005 Read 8 Sent Done Done false
250552023-09-22T23:35:45.194Z 7 Acked 1006 Flush 0 Sent Done Done false
250562023-09-22T23:35:45.197Z 8 Acked 1007 Write 2 Sent Done Done false
250572023-09-22T23:35:45.200Z 9 Acked 1008 Read 8 Sent Done Done false
250582023-09-22T23:35:45.203Z 10 Acked 1009 Flush 0 Sent Done Done false
250592023-09-22T23:35:45.206Z 11 Acked 1010 Write 9 Sent Done Done false
250602023-09-22T23:35:45.208Z 12 Acked 1011 Read 2 Sent Done Done false
250612023-09-22T23:35:45.211Z 13 Acked 1012 Write 6 Sent Done Done false
250622023-09-22T23:35:45.214Z 14 Acked 1013 Write 2 Sent Done Done false
250632023-09-22T23:35:45.217Z 15 Acked 1014 Write 10 Sent Done Done false
250642023-09-22T23:35:45.220Z 16 Acked 1015 Write 5 Sent Done Done false
250652023-09-22T23:35:45.223Z 17 Acked 1016 Write 9 Sent Done Done false
250662023-09-22T23:35:45.225Z 18 Acked 1017 Read 2 Sent Done Done false
250672023-09-22T23:35:45.228Z 19 Acked 1018 Read 8 Sent Done Done false
250682023-09-22T23:35:45.231Z 20 Acked 1019 Write 3 Sent Done Done false
250692023-09-22T23:35:45.234Z 21 Acked 1020 Write 10 Sent Done Done false
250702023-09-22T23:35:45.237Z 22 Acked 1021 Write 2 Sent Done Done false
250712023-09-22T23:35:45.240Z 23 Acked 1022 Read 3 Sent Done Done false
250722023-09-22T23:35:45.243Z 24 Acked 1023 Write 2 Sent Done Done false
250732023-09-22T23:35:45.245Z 25 Acked 1024 Write 2 Sent Done Done false
250742023-09-22T23:35:45.248Z 26 Acked 1025 Read 8 Sent Done Done false
250752023-09-22T23:35:45.251Z 27 Acked 1026 Read 7 Sent Done Done false
250762023-09-22T23:35:45.254Z 28 Acked 1027 Write 3 Sent Done Done false
250772023-09-22T23:35:45.257Z 29 Acked 1028 Write 9 Sent Done Done false
250782023-09-22T23:35:45.259Z 30 Acked 1029 Read 8 Sent Sent Done false
250792023-09-22T23:35:45.262Z STATES DS:0 DS:1 DS:2 TOTAL
250802023-09-22T23:35:45.265Z New 0 0 0 0
250812023-09-22T23:35:45.267Z Sent 28 1 0 29
250822023-09-22T23:35:45.270Z Done 2 29 30 61
250832023-09-22T23:35:45.272Z Skipped 0 0 0 0
250842023-09-22T23:35:45.275Z Error 0 0 0 0
250852023-09-22T23:35:45.278ZLast Flush: 0 1009 1009
250862023-09-22T23:35:45.281ZDownstairs last five completed:
250872023-09-22T23:35:45.283ZUpstairs last five completed: 30 29 28 27 26 25
250882023-09-22T23:35:45.286ZWrote out file "/tmp/test_repair_verify.data"
250892023-09-22T23:35:45.289Z
250902023-09-22T23:35:45.291ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
250912023-09-22T23:35:45.294Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:45.094349901Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1944}
250922023-09-22T23:35:45.297Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:45.094755418Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1944}
250932023-09-22T23:35:45.300Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:45.094780022Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1944}
250942023-09-22T23:35:45.303Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:45.121785992Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1944}
250952023-09-22T23:35:45.306Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:45.121849915Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1944}
250962023-09-22T23:35:45.309Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:45.121881551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1944}
250972023-09-22T23:35:45.312Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:45.149963326Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1944}
250982023-09-22T23:35:45.314Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:45.150021568Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1944}
250992023-09-22T23:35:45.317Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:45.15006157Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1944}
251002023-09-22T23:35:45.320ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
251012023-09-22T23:35:45.322Z 0 000-019 187 187 187 312 312 312 F T T <---
251022023-09-22T23:35:45.325Z 1 020-039 184 184 184 308 308 308 F F F
251032023-09-22T23:35:45.328Z 2 040-059 188 190 190 315 317 317 F F F <---
251042023-09-22T23:35:45.331Z 3 060-079 188 188 188 315 315 315 F F F
251052023-09-22T23:35:45.333Z 4 080-099 180 180 180 291 291 291 F F F
251062023-09-22T23:35:45.336Z 5 100-119 185 185 185 309 309 309 F T T <---
251072023-09-22T23:35:45.339Z 6 120-139 184 184 184 305 305 305 F F F
251082023-09-22T23:35:45.342Z 7 140-159 186 186 186 311 311 311 F T T <---
251092023-09-22T23:35:45.344Z 8 160-179 188 188 188 314 314 314 F F F
251102023-09-22T23:35:45.347Z 9 180-199 189 189 189 316 316 316 F F F
251112023-09-22T23:35:45.350Z 10 200-219 189 189 189 316 316 316 F F F
251122023-09-22T23:35:45.353Z 11 220-239 188 188 188 315 315 315 F F F
251132023-09-22T23:35:45.356Z 12 240-259 188 188 188 315 315 315 F F F
251142023-09-22T23:35:45.358Z 13 260-279 188 190 190 313 317 317 T F F <---
251152023-09-22T23:35:45.361Z 14 280-299 184 184 184 305 305 305 F F F
251162023-09-22T23:35:45.363ZError: Difference in extent metadata found!
251172023-09-22T23:35:45.366Z 15 300-319 187 187 187 312 312 312 F F F
251182023-09-22T23:35:45.369Z 16 320-339 187 187 187 312 312 312 F T T <---
251192023-09-22T23:35:45.372Z 17 340-359 184 184 184 305 305 305 F T T <---
251202023-09-22T23:35:45.374Z 18 360-379 189 189 189 316 316 316 F F F
251212023-09-22T23:35:45.377Z 19 380-399 188 188 188 315 315 315 F F F
251222023-09-22T23:35:45.380Z 20 400-419 186 186 186 311 311 311 F T T <---
251232023-09-22T23:35:45.383Z 21 420-439 184 184 184 306 306 306 F T T <---
251242023-09-22T23:35:45.385Z 22 440-459 188 188 188 315 315 315 F T T <---
251252023-09-22T23:35:45.388Z 23 460-479 188 188 188 315 315 315 F T T <---
251262023-09-22T23:35:45.390Z 24 480-499 187 187 187 312 312 312 F T T <---
251272023-09-22T23:35:45.393Z 25 500-519 187 190 190 312 318 318 F F F <---
251282023-09-22T23:35:45.396Z 26 520-539 186 186 186 311 311 311 F F F
251292023-09-22T23:35:45.399Z 27 540-559 188 188 188 313 313 313 F T T <---
251302023-09-22T23:35:45.401Z 28 560-579 179 179 179 290 290 290 F T T <---
251312023-09-22T23:35:45.404Z 29 580-599 188 188 188 313 313 313 F F F
251322023-09-22T23:35:45.407ZMax gen: 190, Max flush: 318
251332023-09-22T23:35:45.409ZA Difference in extent metadata is expected here
251342023-09-22T23:35:45.412ZOn loop 95
251352023-09-22T23:35:45.415Z
251362023-09-22T23:35:45.418ZVerifying data now
251372023-09-22T23:35:51.814ZNo change after verify
251382023-09-22T23:35:51.817ZLoop: 95 Downstairs dump after verify (and repair):
251392023-09-22T23:35:51.832Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.830145436Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1949}
251402023-09-22T23:35:51.834Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.830550432Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1949}
251412023-09-22T23:35:51.837Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.830575684Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1949}
251422023-09-22T23:35:51.851Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.849620491Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1949}
251432023-09-22T23:35:51.853Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.849673398Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1949}
251442023-09-22T23:35:51.856Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.849694148Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1949}
251452023-09-22T23:35:51.862Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.860831281Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1949}
251462023-09-22T23:35:51.864Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.860884937Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1949}
251472023-09-22T23:35:51.867Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.860905889Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1949}
251482023-09-22T23:35:51.873ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
251492023-09-22T23:35:51.876Z 0 000-019 191 191 191 319 319 319 F F F
251502023-09-22T23:35:51.878Z 1 020-039 184 184 184 308 308 308 F F F
251512023-09-22T23:35:51.881Z 2 040-059 190 190 190 317 317 317 F F F
251522023-09-22T23:35:51.884Z 3 060-079 188 188 188 315 315 315 F F F
251532023-09-22T23:35:51.886Z 4 080-099 180 180 180 291 291 291 F F F
251542023-09-22T23:35:51.889Z 5 100-119 191 191 191 319 319 319 F F F
251552023-09-22T23:35:51.892Z 6 120-139 184 184 184 305 305 305 F F F
251562023-09-22T23:35:51.894Z 7 140-159 191 191 191 319 319 319 F F F
251572023-09-22T23:35:51.897Z 8 160-179 188 188 188 314 314 314 F F F
251582023-09-22T23:35:51.900Z 9 180-199 189 189 189 316 316 316 F F F
251592023-09-22T23:35:51.902Z 10 200-219 189 189 189 316 316 316 F F F
251602023-09-22T23:35:51.905Z 11 220-239 188 188 188 315 315 315 F F F
251612023-09-22T23:35:51.908Z 12 240-259 188 188 188 315 315 315 F F F
251622023-09-22T23:35:51.911Z 13 260-279 190 190 190 317 317 317 F F F
251632023-09-22T23:35:51.913Z 14 280-299 184 184 184 305 305 305 F F F
251642023-09-22T23:35:51.917Z 15 300-319 187 187 187 312 312 312 F F F
251652023-09-22T23:35:51.920Z 16 320-339 191 191 191 319 319 319 F F F
251662023-09-22T23:35:51.923Z 17 340-359 191 191 191 319 319 319 F F F
251672023-09-22T23:35:51.925Z 18 360-379 189 189 189 316 316 316 F F F
251682023-09-22T23:35:51.928Z 19 380-399 188 188 188 315 315 315 F F F
251692023-09-22T23:35:51.931Z 20 400-419 191 191 191 319 319 319 F F F
251702023-09-22T23:35:51.933Z 21 420-439 191 191 191 319 319 319 F F F
251712023-09-22T23:35:51.936Z 22 440-459 191 191 191 319 319 319 F F F
251722023-09-22T23:35:51.939Z 23 460-479 191 191 191 319 319 319 F F F
251732023-09-22T23:35:51.941Z 24 480-499 191 191 191 319 319 319 F F F
251742023-09-22T23:35:51.944Z 25 500-519 190 190 190 318 318 318 F F F
251752023-09-22T23:35:51.947Z 26 520-539 186 186 186 311 311 311 F F F
251762023-09-22T23:35:51.950Z 27 540-559 191 191 191 319 319 319 F F F
251772023-09-22T23:35:51.953Z 28 560-579 191 191 191 319 319 319 F F F
251782023-09-22T23:35:51.956Z 29 580-599 188 188 188 313 313 313 F F F
251792023-09-22T23:35:51.958ZMax gen: 191, Max flush: 319
251802023-09-22T23:35:51.961Z
251812023-09-22T23:35:51.964ZBegin loop 96
251822023-09-22T23:35:51.966ZDownstairs to restart: 2
251832023-09-22T23:35:51.969Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.911541256Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
251842023-09-22T23:35:51.972Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.911926207Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
251852023-09-22T23:35:51.975Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.911995209Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
251862023-09-22T23:35:51.978Z{"msg":"Crucible stats registered with UUID: 6807e0e4-5098-4933-b24c-62967316c7b9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.912111729Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
251872023-09-22T23:35:51.980Z{"msg":"Crucible 6807e0e4-5098-4933-b24c-62967316c7b9 has session id: 0d00ab18-0838-4cd8-b09c-d070a366ce7a","v":0,"name":"Crucible runtime is spawned
251882023-09-22T23:35:51.983Zcrucible","level":30The guest has requested activation with gen:192
251892023-09-22T23:35:51.986Z,"time":"2023-09-22T23:35:51.912240167Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
251902023-09-22T23:35:51.989Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.912343103Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951,"task":"up_listen"}
251912023-09-22T23:35:51.992Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.912416232Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
251922023-09-22T23:35:51.995Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.912487867Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
251932023-09-22T23:35:51.998Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.91258429Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951,"looper":"1"}
251942023-09-22T23:35:52.001Z{"msg":"[1] 6807e0e4-5098-4933-b24c-62967316c7b9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.912864595Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951,"looper":"1"}
251952023-09-22T23:35:52.004Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.912934924Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951,"looper":"0"}
251962023-09-22T23:35:52.007Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.913027076Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951,"looper":"2"}
251972023-09-22T23:35:52.009Z{"msg":"[0] 6807e0e4-5098-4933-b24c-62967316c7b9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.913085565Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951,"looper":"0"}
251982023-09-22T23:35:52.012Z{"msg":"6807e0e4-5098-4933-b24c-62967316c7b9 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.913545399Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
251992023-09-22T23:35:52.015Z{"msg":"Set desired generation to :192","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.913603606Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252002023-09-22T23:35:52.017Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.913704939Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252012023-09-22T23:35:52.020Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.914008924Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252022023-09-22T23:35:52.023Z{"msg":"[1] 6807e0e4-5098-4933-b24c-62967316c7b9 (e202fc05-6cd5-4779-8d21-fa4754e158aa) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.914397829Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252032023-09-22T23:35:52.025Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.914453898Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252042023-09-22T23:35:52.028Z{"msg":"[1] client is_active_req TRUE, promote! session e202fc05-6cd5-4779-8d21-fa4754e158aa","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.914512324Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252052023-09-22T23:35:52.030Z{"msg":"[0] 6807e0e4-5098-4933-b24c-62967316c7b9 (e202fc05-6cd5-4779-8d21-fa4754e158aa) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.914627365Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252062023-09-22T23:35:52.033Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.914786426Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252072023-09-22T23:35:52.036Z{"msg":"[0] client is_active_req TRUE, promote! session e202fc05-6cd5-4779-8d21-fa4754e158aa","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.914843124Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252082023-09-22T23:35:52.039Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.915143671Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252092023-09-22T23:35:52.041Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.915236466Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252102023-09-22T23:35:52.044Z{"msg":"6807e0e4-5098-4933-b24c-62967316c7b9 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.915306189Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252112023-09-22T23:35:52.047Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.915372069Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252122023-09-22T23:35:52.049Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.915421934Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252132023-09-22T23:35:52.052Z{"msg":"6807e0e4-5098-4933-b24c-62967316c7b9 WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.915461949Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252142023-09-22T23:35:52.054Z{"msg":"[1] 6807e0e4-5098-4933-b24c-62967316c7b9 (e202fc05-6cd5-4779-8d21-fa4754e158aa) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.917449737Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252152023-09-22T23:35:52.078Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.917507438Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252162023-09-22T23:35:52.081Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:35:51.91753135Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252172023-09-22T23:35:52.084Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.917583135Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252182023-09-22T23:35:52.087Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.917722876Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252192023-09-22T23:35:52.089Z{"msg":"6807e0e4-5098-4933-b24c-62967316c7b9 WaitActive WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.91777668Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252202023-09-22T23:35:52.092Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.9177987Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252212023-09-22T23:35:52.095Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.917847232Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252222023-09-22T23:35:52.097Z{"msg":"[0] 6807e0e4-5098-4933-b24c-62967316c7b9 (e202fc05-6cd5-4779-8d21-fa4754e158aa) WaitActive WaitQuorum New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.917892346Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252232023-09-22T23:35:52.100Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.917932253Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252242023-09-22T23:35:52.102Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:35:51.917953421Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252252023-09-22T23:35:52.105Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.917989589Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252262023-09-22T23:35:52.108Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.918029449Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252272023-09-22T23:35:52.110Z{"msg":"6807e0e4-5098-4933-b24c-62967316c7b9 WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.918067857Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252282023-09-22T23:35:52.113Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.918087927Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252292023-09-22T23:35:52.116Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:51.918129727Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252302023-09-22T23:35:52.917Z{"msg":"[2] 6807e0e4-5098-4933-b24c-62967316c7b9 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.915068643Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951,"looper":"2"}
252312023-09-22T23:35:52.920Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.915154395Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252322023-09-22T23:35:52.923Z{"msg":"[2] 6807e0e4-5098-4933-b24c-62967316c7b9 (e202fc05-6cd5-4779-8d21-fa4754e158aa) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.916110148Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252332023-09-22T23:35:52.925Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.916167689Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252342023-09-22T23:35:52.928Z{"msg":"[2] client is_active_req TRUE, promote! session e202fc05-6cd5-4779-8d21-fa4754e158aa","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.916211603Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252352023-09-22T23:35:52.931Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.916788552Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252362023-09-22T23:35:52.934Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.916845628Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252372023-09-22T23:35:52.937Z{"msg":"6807e0e4-5098-4933-b24c-62967316c7b9 WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.916889531Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252382023-09-22T23:35:52.940Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.919132784Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252392023-09-22T23:35:52.943Z{"msg":"[2] 6807e0e4-5098-4933-b24c-62967316c7b9 (e202fc05-6cd5-4779-8d21-fa4754e158aa) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.91930597Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252402023-09-22T23:35:52.945Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.919378329Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252412023-09-22T23:35:52.948Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:35:52.919439727Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252422023-09-22T23:35:52.951Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.919484103Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252432023-09-22T23:35:52.954Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.919523631Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252442023-09-22T23:35:52.957Z{"msg":"6807e0e4-5098-4933-b24c-62967316c7b9 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.919564118Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252452023-09-22T23:35:52.960Z{"msg":"[0]R flush_numbers[0..12]: [319, 308, 317, 315, 291, 319, 305, 319, 314, 316, 316, 315]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.919612751Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252462023-09-22T23:35:52.963Z{"msg":"[0]R generation[0..12]: [191, 184, 190, 188, 180, 191, 184, 191, 188, 189, 189, 188]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.919653409Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252472023-09-22T23:35:52.965ZThe guest has finished waiting for activation with:192
252482023-09-22T23:35:52.968Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"Wait for a query_work_queue command to finish before sending IO
252492023-09-22T23:35:52.971Zname":"crucible","level":30,"time":"2023-09-22T23:35:52.919684207Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252502023-09-22T23:35:52.974Z{"msg":"[1]R flush_numbers[0..12]: [319, 308, 317, 315, 291, 319, 305, 319, 314, 316, 316, 315]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.919738035Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252512023-09-22T23:35:52.977Z{"msg":"[1]R generation[0..12]: [191, 184, 190, 188, 180, 191, 184, 191, 188, 189, 189, 188]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.919779926Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252522023-09-22T23:35:52.980Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.919820638Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252532023-09-22T23:35:52.983Z{"msg":"[2]R flush_numbers[0..12]: [319, 308, 317, 315, 291, 319, 305, 319, 314, 316, 316, 315]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.919868691Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252542023-09-22T23:35:52.986Z{"msg":"[2]R generation[0..12]: [191, 184, 190, 188, 180, 191, 184, 191, 188, 189, 189, 188]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.919926437Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252552023-09-22T23:35:52.989Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.920004252Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252562023-09-22T23:35:52.992Z{"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.920068394Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252572023-09-22T23:35:52.999Z{"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.920141208Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252582023-09-22T23:35:53.002Z{"msg":"Next flush: 320","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.92020422Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252592023-09-22T23:35:53.005Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.92027446Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252602023-09-22T23:35:53.008Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.920337197Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252612023-09-22T23:35:53.011Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.920405314Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252622023-09-22T23:35:53.014Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.920468319Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252632023-09-22T23:35:53.017Z{"msg":"6807e0e4-5098-4933-b24c-62967316c7b9 is now active with session: e202fc05-6cd5-4779-8d21-fa4754e158aa","v":0,"name":"crucible","level":30,"time":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
252642023-09-22T23:35:53.020Z2023-09-22T23:35:52.920525891Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252652023-09-22T23:35:53.023Z{"msg":"6807e0e4-5098-4933-b24c-62967316c7b9 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.920661436Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252662023-09-22T23:35:53.026Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.920733541Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252672023-09-22T23:35:53.028Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.920797016Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252682023-09-22T23:35:53.031Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.920861651Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252692023-09-22T23:35:53.034Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.920923733Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252702023-09-22T23:35:53.037Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.920984201Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252712023-09-22T23:35:53.039Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.921044309Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252722023-09-22T23:35:53.042Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.921104692Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252732023-09-22T23:35:53.045Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.92116632Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252742023-09-22T23:35:53.048Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.921227276Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252752023-09-22T23:35:53.050Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.921287243Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252762023-09-22T23:35:53.053Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:52.921346901Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1951}
252772023-09-22T23:35:53.056ZLoading write count information from file "/tmp/test_repair_verify.data"
252782023-09-22T23:35:53.058ZRun Repair workload
252792023-09-22T23:35:53.061Z01/30 Write block 149 len 40960 data: 16 16 16 15 11 13 14 13 14 16
252802023-09-22T23:35:53.064Z02/30 Write block 300 len 12288 data: 12 14 16
252812023-09-22T23:35:53.067Z03/30 Read block 246 len 4096
252822023-09-22T23:35:53.070Z04/30 Write block 597 len 12288 data: 7 7 3
252832023-09-22T23:35:53.072Z05/30 Write block 229 len 24576 data: 10 10 7 9 10 12
252842023-09-22T23:35:53.075Z06/30 Read block 86 len 8192
252852023-09-22T23:35:53.078Z07/30 Write block 509 len 32768 data: 15 16 14 18 20 23 21 19
252862023-09-22T23:35:53.101Z08/30 Read block 381 len 32768
252872023-09-22T23:35:53.104Z09/30 Read block 413 len 16384
252882023-09-22T23:35:53.107Z10/30 Write block 418 len 40960 data: 17 19 17 16 13 14 13 9 13 15
252892023-09-22T23:35:53.110Z11/30 Flush
252902023-09-22T23:35:53.113Z12/30 Write block 269 len 4096 data: 17
252912023-09-22T23:35:53.115Z13/30 Write block 512 len 4096 data: 19
252922023-09-22T23:35:53.118Z14/30 Write block 221 len 8192 data: 11 10
252932023-09-22T23:35:53.121Z15/30 Read block 406 len 16384
252942023-09-22T23:35:53.124Z16/30 Read block 228 len 24576
252952023-09-22T23:35:53.126Z17/30 Flush
252962023-09-22T23:35:53.129Z18/30 Write block 523 len 40960 data: 15 16 19 19 21 18 16 14 14 15
252972023-09-22T23:35:53.132Z19/30 Read block 23 len 40960
252982023-09-22T23:35:53.134Z20/30 Write block 8 len 8192 data: 11 11
252992023-09-22T23:35:53.137Z21/30 Flush
253002023-09-22T23:35:53.140Z22/30 Write block 531 len 24576 data: 15 16 18 14 16 16
253012023-09-22T23:35:53.142Z23/30 Write block 123 len 8192 data: 15 13
253022023-09-22T23:35:53.145Z24/30 Read block 390 len 12288
253032023-09-22T23:35:53.148Z25/30 Read block 77 len 40960
253042023-09-22T23:35:53.151Z26/30 Write block 401 len 36864 data: 16 17 16 16 14 11 12 11 13
253052023-09-22T23:35:53.154Z27/30 Read block 174 len 40960
253062023-09-22T23:35:53.156Z28/30 Write block 114 len 24576 data: 18 14 14 11 12 12
253072023-09-22T23:35:53.160Z29/30 Write block 470 len 32768 data: 15 13 14 13 14 13 12 11
253082023-09-22T23:35:53.163Z30/30 Read block 202 len 4096
253092023-09-22T23:35:53.166Z----------------------------------------------------------------
253102023-09-22T23:35:53.169Z Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30
253112023-09-22T23:35:53.171ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
253122023-09-22T23:35:53.174Z 1 Acked 1000 Write 10 Done Done Done false
253132023-09-22T23:35:53.177Z 2 Acked 1001 Write 3 Done Done Done false
253142023-09-22T23:35:53.180Z 3 Acked 1002 Read 1 Done Done Done false
253152023-09-22T23:35:53.182Z 4 Acked 1003 Write 3 Done Done Done false
253162023-09-22T23:35:53.185Z 5 Acked 1004 Write 6 Done Done Done false
253172023-09-22T23:35:53.188Z 6 Acked 1005 Read 2 Done Done Done false
253182023-09-22T23:35:53.191Z 7 Acked 1006 Write 8 Done Done Sent false
253192023-09-22T23:35:53.193Z 8 Acked 1007 Read 8 Done Done Sent false
253202023-09-22T23:35:53.199Z 9 Acked 1008 Read 4 Done Done Sent false
253212023-09-22T23:35:53.203Z 10 Acked 1009 Write 10 Done Done Sent false
253222023-09-22T23:35:53.206Z 11 Acked 1010 Flush 0 Done Done Sent false
253232023-09-22T23:35:53.209Z 12 Acked 1011 Write 1 Done Done Sent false
253242023-09-22T23:35:53.211Z 13 Acked 1012 Write 1 Done Done Sent false
253252023-09-22T23:35:53.214Z 14 Acked 1013 Write 2 Done Done Sent false
253262023-09-22T23:35:53.217Z 15 Acked 1014 Read 4 Done Done Sent false
253272023-09-22T23:35:53.219Z 16 Acked 1015 Read 6 Done Done Sent false
253282023-09-22T23:35:53.222Z 17 Acked 1016 Flush 0 Done Done Sent false
253292023-09-22T23:35:53.225Z 18 Acked 1017 Write 10 Done Done Sent false
253302023-09-22T23:35:53.228Z 19 Acked 1018 Read 10 Done Done Sent false
253312023-09-22T23:35:53.231Z 20 Acked 1019 Write 2 Done Done Sent false
253322023-09-22T23:35:53.234Z 21 Acked 1020 Flush 0 Done Done Sent false
253332023-09-22T23:35:53.236Z 22 Acked 1021 Write 6 Done Done Sent false
253342023-09-22T23:35:53.239Z 23 Acked 1022 Write 2 Done Done Sent false
253352023-09-22T23:35:53.242Z 24 Acked 1023 Read 3 Done Done Sent false
253362023-09-22T23:35:53.244Z 25 Acked 1024 Read 10 Done Done Sent false
253372023-09-22T23:35:53.247Z 26 Acked 1025 Write 9 Done Done Sent false
253382023-09-22T23:35:53.250Z 27 Acked 1026 Read 10 Done Done Sent false
253392023-09-22T23:35:53.252Z 28 Acked 1027 Write 6 Done Done Sent false
253402023-09-22T23:35:53.255Z 29 Acked 1028 Write 8 Done Sent Sent false
253412023-09-22T23:35:53.257Z 30 Acked 1029 Read 1 Done Sent Sent false
253422023-09-22T23:35:53.260Z STATES DS:0 DS:1 DS:2 TOTAL
253432023-09-22T23:35:53.263Z New 0 0 0 0
253442023-09-22T23:35:53.265Z Sent 0 2 24 26
253452023-09-22T23:35:53.268Z Done 30 28 6 64
253462023-09-22T23:35:53.271Z Skipped 0 0 0 0
253472023-09-22T23:35:53.274Z Error 0 0 0 0
253482023-09-22T23:35:53.277ZLast Flush: 1020 1020 0
253492023-09-22T23:35:53.279ZDownstairs last five completed:
253502023-09-22T23:35:53.282ZUpstairs last five completed: 30 29 28 27 26 25
253512023-09-22T23:35:53.285ZWrote out file "/tmp/test_repair_verify.data"
253522023-09-22T23:35:53.288Z
253532023-09-22T23:35:53.291ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
253542023-09-22T23:35:53.295Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:53.060744714Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1952}
253552023-09-22T23:35:53.298Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:53.06106362Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1952}
253562023-09-22T23:35:53.300Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:53.061086805Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1952}
253572023-09-22T23:35:53.303Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:53.08852071Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1952}
253582023-09-22T23:35:53.306Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:53.088554103Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1952}
253592023-09-22T23:35:53.309Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:53.08857341Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1952}
253602023-09-22T23:35:53.311Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:53.106285256Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1952}
253612023-09-22T23:35:53.314Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:53.1063321Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1952}
253622023-09-22T23:35:53.317Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:53.106370564Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1952}
253632023-09-22T23:35:53.320ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
253642023-09-22T23:35:53.323Z 0 000-019 192 192 191 322 322 319 F F F <---
253652023-09-22T23:35:53.325Z 1 020-039 184 184 184 308 308 308 F F F
253662023-09-22T23:35:53.328ZError: Difference in extent metadata found!
253672023-09-22T23:35:53.331Z 2 040-059 190 190 190 317 317 317 F F F
253682023-09-22T23:35:53.334Z 3 060-079 188 188 188 315 315 315 F F F
253692023-09-22T23:35:53.336Z 4 080-099 180 180 180 291 291 291 F F F
253702023-09-22T23:35:53.340Z 5 100-119 191 191 191 319 319 319 T T F <---
253712023-09-22T23:35:53.343Z 6 120-139 184 184 184 305 305 305 T T F <---
253722023-09-22T23:35:53.345Z 7 140-159 192 192 191 320 320 319 F F T <---
253732023-09-22T23:35:53.348Z 8 160-179 188 188 188 314 314 314 F F F
253742023-09-22T23:35:53.351Z 9 180-199 189 189 189 316 316 316 F F F
253752023-09-22T23:35:53.353Z 10 200-219 189 189 189 316 316 316 F F F
253762023-09-22T23:35:53.357Z 11 220-239 192 192 188 321 321 315 F F T <---
253772023-09-22T23:35:53.360Z 12 240-259 188 188 188 315 315 315 F F F
253782023-09-22T23:35:53.363Z 13 260-279 192 192 190 321 321 317 F F F <---
253792023-09-22T23:35:53.365Z 14 280-299 184 184 184 305 305 305 F F F
253802023-09-22T23:35:53.368Z 15 300-319 192 192 187 320 320 312 F F T <---
253812023-09-22T23:35:53.371Z 16 320-339 191 191 191 319 319 319 F F F
253822023-09-22T23:35:53.374Z 17 340-359 191 191 191 319 319 319 F F F
253832023-09-22T23:35:53.376Z 18 360-379 189 189 189 316 316 316 F F F
253842023-09-22T23:35:53.379Z 19 380-399 188 188 188 315 315 315 F F F
253852023-09-22T23:35:53.382Z 20 400-419 192 192 191 320 320 319 T T F <---
253862023-09-22T23:35:53.385Z 21 420-439 192 192 191 320 320 319 F F F <---
253872023-09-22T23:35:53.388Z 22 440-459 191 191 191 319 319 319 F F F
253882023-09-22T23:35:53.391Z 23 460-479 191 191 191 319 319 319 T T F <---
253892023-09-22T23:35:53.393Z 24 480-499 191 191 191 319 319 319 F F F
253902023-09-22T23:35:53.396Z 25 500-519 192 192 190 321 321 318 F F F <---
253912023-09-22T23:35:53.399Z 26 520-539 192 192 186 322 322 311 T T F <---
253922023-09-22T23:35:53.401Z 27 540-559 191 191 191 319 319 319 F F F
253932023-09-22T23:35:53.404Z 28 560-579 191 191 191 319 319 319 F F F
253942023-09-22T23:35:53.407Z 29 580-599 192 192 188 320 320 313 F F T <---
253952023-09-22T23:35:53.410ZMax gen: 192, Max flush: 322
253962023-09-22T23:35:53.413ZA Difference in extent metadata is expected here
253972023-09-22T23:35:53.416ZOn loop 96
253982023-09-22T23:35:53.421Z
253992023-09-22T23:35:53.425ZVerifying data now
254002023-09-22T23:35:59.608ZNo change after verify
254012023-09-22T23:35:59.611ZLoop: 96 Downstairs dump after verify (and repair):
254022023-09-22T23:35:59.626Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.624954808Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1957}
254032023-09-22T23:35:59.630Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.625246251Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1957}
254042023-09-22T23:35:59.633Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.625270968Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1957}
254052023-09-22T23:35:59.645Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.643938877Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1957}
254062023-09-22T23:35:59.648Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.643993408Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1957}
254072023-09-22T23:35:59.650Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.644014135Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1957}
254082023-09-22T23:35:59.656Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.654858729Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1957}
254092023-09-22T23:35:59.658Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.654909646Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1957}
254102023-09-22T23:35:59.661Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.654930166Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1957}
254112023-09-22T23:35:59.667ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
254122023-09-22T23:35:59.670Z 0 000-019 192 192 192 322 322 322 F F F
254132023-09-22T23:35:59.673Z 1 020-039 184 184 184 308 308 308 F F F
254142023-09-22T23:35:59.676Z 2 040-059 190 190 190 317 317 317 F F F
254152023-09-22T23:35:59.678Z 3 060-079 188 188 188 315 315 315 F F F
254162023-09-22T23:35:59.681Z 4 080-099 180 180 180 291 291 291 F F F
254172023-09-22T23:35:59.684Z 5 100-119 193 193 193 323 323 323 F F F
254182023-09-22T23:35:59.687Z 6 120-139 193 193 193 323 323 323 F F F
254192023-09-22T23:35:59.689Z 7 140-159 192 192 192 320 320 320 F F F
254202023-09-22T23:35:59.692Z 8 160-179 188 188 188 314 314 314 F F F
254212023-09-22T23:35:59.694Z 9 180-199 189 189 189 316 316 316 F F F
254222023-09-22T23:35:59.697Z 10 200-219 189 189 189 316 316 316 F F F
254232023-09-22T23:35:59.700Z 11 220-239 192 192 192 321 321 321 F F F
254242023-09-22T23:35:59.703Z 12 240-259 188 188 188 315 315 315 F F F
254252023-09-22T23:35:59.706Z 13 260-279 192 192 192 321 321 321 F F F
254262023-09-22T23:35:59.709Z 14 280-299 184 184 184 305 305 305 F F F
254272023-09-22T23:35:59.711Z 15 300-319 192 192 192 320 320 320 F F F
254282023-09-22T23:35:59.714Z 16 320-339 191 191 191 319 319 319 F F F
254292023-09-22T23:35:59.717Z 17 340-359 191 191 191 319 319 319 F F F
254302023-09-22T23:35:59.721Z 18 360-379 189 189 189 316 316 316 F F F
254312023-09-22T23:35:59.724Z 19 380-399 188 188 188 315 315 315 F F F
254322023-09-22T23:35:59.727Z 20 400-419 193 193 193 323 323 323 F F F
254332023-09-22T23:35:59.730Z 21 420-439 192 192 192 320 320 320 F F F
254342023-09-22T23:35:59.733Z 22 440-459 191 191 191 319 319 319 F F F
254352023-09-22T23:35:59.736Z 23 460-479 193 193 193 323 323 323 F F F
254362023-09-22T23:35:59.738Z 24 480-499 191 191 191 319 319 319 F F F
254372023-09-22T23:35:59.742Z 25 500-519 192 192 192 321 321 321 F F F
254382023-09-22T23:35:59.744Z 26 520-539 193 193 193 323 323 323 F F F
254392023-09-22T23:35:59.747Z 27 540-559 191 191 191 319 319 319 F F F
254402023-09-22T23:35:59.750Z 28 560-579 191 191 191 319 319 319 F F F
254412023-09-22T23:35:59.753Z 29 580-599 192 192 192 320 320 320 F F F
254422023-09-22T23:35:59.755ZMax gen: 193, Max flush: 323
254432023-09-22T23:35:59.758Z
254442023-09-22T23:35:59.761ZBegin loop 97
254452023-09-22T23:35:59.764ZDownstairs to restart: 0
254462023-09-22T23:35:59.766Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.70303029Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254472023-09-22T23:35:59.769Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.703398057Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254482023-09-22T23:35:59.772Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.703449545Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254492023-09-22T23:35:59.775Z{"msg":"Crucible stats registered with UUID: 7a8c4a1e-9bb9-4923-837a-7438fb102f98","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.703475271Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254502023-09-22T23:35:59.777Z{"msg":"Crucible 7a8c4a1e-9bb9-4923-837a-7438fb102f98 has session id: 48576ebf-2ef3-4aee-87f3-40c792d6ef22","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.703516952Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254512023-09-22T23:35:59.780ZCrucible runtime is spawned
254522023-09-22T23:35:59.783Z{"msg":"up_listen starts","v":0,"name"The guest has requested activation with gen:194
254532023-09-22T23:35:59.786Z:"crucible","level":30,"time":"2023-09-22T23:35:59.703908751Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959,"task":"up_listen"}
254542023-09-22T23:35:59.788Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.703989038Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254552023-09-22T23:35:59.791Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.704056935Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254562023-09-22T23:35:59.793Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.704143012Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959,"looper":"1"}
254572023-09-22T23:35:59.796Z{"msg":"7a8c4a1e-9bb9-4923-837a-7438fb102f98 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.704217522Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254582023-09-22T23:35:59.799Z{"msg":"Set desired generation to :194","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.704270844Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254592023-09-22T23:35:59.802Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.704325541Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959,"looper":"2"}
254602023-09-22T23:35:59.804Z{"msg":"[1] 7a8c4a1e-9bb9-4923-837a-7438fb102f98 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.704398197Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959,"looper":"1"}
254612023-09-22T23:35:59.807Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.704451661Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959,"looper":"0"}
254622023-09-22T23:35:59.809Z{"msg":"[2] 7a8c4a1e-9bb9-4923-837a-7438fb102f98 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.704505813Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959,"looper":"2"}
254632023-09-22T23:35:59.817Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.704776719Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254642023-09-22T23:35:59.821Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.704927418Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254652023-09-22T23:35:59.823Z{"msg":"[2] 7a8c4a1e-9bb9-4923-837a-7438fb102f98 (0d46d172-b8c5-4227-a373-9ee85552cc9b) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.705615609Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254662023-09-22T23:35:59.826Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.705707414Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254672023-09-22T23:35:59.829Z{"msg":"[2] client is_active_req TRUE, promote! session 0d46d172-b8c5-4227-a373-9ee85552cc9b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.705752537Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254682023-09-22T23:35:59.832Z{"msg":"[1] 7a8c4a1e-9bb9-4923-837a-7438fb102f98 (0d46d172-b8c5-4227-a373-9ee85552cc9b) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.705814205Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254692023-09-22T23:35:59.835Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.705929435Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254702023-09-22T23:35:59.838Z{"msg":"[1] client is_active_req TRUE, promote! session 0d46d172-b8c5-4227-a373-9ee85552cc9b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.705991721Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254712023-09-22T23:35:59.840Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.706482226Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254722023-09-22T23:35:59.843Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.706536626Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254732023-09-22T23:35:59.846Z{"msg":"7a8c4a1e-9bb9-4923-837a-7438fb102f98 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.706692831Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254742023-09-22T23:35:59.849Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.706820902Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254752023-09-22T23:35:59.852Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.706893597Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254762023-09-22T23:35:59.855Z{"msg":"7a8c4a1e-9bb9-4923-837a-7438fb102f98 New WaitActive WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.706937245Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254772023-09-22T23:35:59.858Z{"msg":"[2] 7a8c4a1e-9bb9-4923-837a-7438fb102f98 (0d46d172-b8c5-4227-a373-9ee85552cc9b) New WaitActive WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.708883893Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254782023-09-22T23:35:59.860Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.709004999Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254792023-09-22T23:35:59.863Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:35:59.709079241Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254802023-09-22T23:35:59.867Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.709134849Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254812023-09-22T23:35:59.870Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.709176443Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254822023-09-22T23:35:59.872Z{"msg":"7a8c4a1e-9bb9-4923-837a-7438fb102f98 New WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.709215379Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254832023-09-22T23:35:59.875Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.70925351Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254842023-09-22T23:35:59.878Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.70929207Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254852023-09-22T23:35:59.881Z{"msg":"[1] 7a8c4a1e-9bb9-4923-837a-7438fb102f98 (0d46d172-b8c5-4227-a373-9ee85552cc9b) New WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.709333835Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254862023-09-22T23:35:59.883Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.709372955Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254872023-09-22T23:35:59.886Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:35:59.709411041Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254882023-09-22T23:35:59.889Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.709448696Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254892023-09-22T23:35:59.892Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.709487723Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254902023-09-22T23:35:59.894Z{"msg":"7a8c4a1e-9bb9-4923-837a-7438fb102f98 New WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.709526445Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254912023-09-22T23:35:59.898Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.709564467Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254922023-09-22T23:35:59.902Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:35:59.709601908Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254932023-09-22T23:36:00.708Z{"msg":"[0] 7a8c4a1e-9bb9-4923-837a-7438fb102f98 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.706617387Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959,"looper":"0"}
254942023-09-22T23:36:00.711Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.706723429Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254952023-09-22T23:36:00.715Z{"msg":"[0] 7a8c4a1e-9bb9-4923-837a-7438fb102f98 (0d46d172-b8c5-4227-a373-9ee85552cc9b) New WaitQuorum WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.707882541Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254962023-09-22T23:36:00.718Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.707938118Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254972023-09-22T23:36:00.720Z{"msg":"[0] client is_active_req TRUE, promote! session 0d46d172-b8c5-4227-a373-9ee85552cc9b","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.707981368Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254982023-09-22T23:36:00.723Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.708563423Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
254992023-09-22T23:36:00.726Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.708668596Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255002023-09-22T23:36:00.729Z{"msg":"7a8c4a1e-9bb9-4923-837a-7438fb102f98 WaitActive WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.708730547Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255012023-09-22T23:36:00.732Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.710845557Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255022023-09-22T23:36:00.735Z{"msg":"[0] 7a8c4a1e-9bb9-4923-837a-7438fb102f98 (0d46d172-b8c5-4227-a373-9ee85552cc9b) WaitActive WaitQuorum WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.711127057Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255032023-09-22T23:36:00.737Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.711185662Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255042023-09-22T23:36:00.740Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:36:00.711228067Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255052023-09-22T23:36:00.743Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.711267304Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255062023-09-22T23:36:00.746Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.711306833Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255072023-09-22T23:36:00.749Z{"msg":"7a8c4a1e-9bb9-4923-837a-7438fb102f98 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.711347011Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255082023-09-22T23:36:00.752Z{"msg":"[0]R flush_numbers[0..12]: [322, 308, 317, 315, 291, 323, 323, 320, 314, 316, 316, 321]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.71138758Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255092023-09-22T23:36:00.755Z{"msg":"[0]R generation[0..12]: [192, 184, 190, 188, 180, 193, 193, 192, 188, 189, 189, 192]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.711428074Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255102023-09-22T23:36:00.758Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.711482238Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255112023-09-22T23:36:00.761Z{"msg":"[1]R flush_numbers[0..12]: [322, 308, 317, 315, 291, 323, 323, 320, 314, 316, 316, 321]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.711503795Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255122023-09-22T23:36:00.764Z{"The guest has finished waiting for activation with:194
255132023-09-22T23:36:00.767ZWait for a query_work_queue command to finish before sending IO
255142023-09-22T23:36:00.770Zmsg":"[1]R generation[0..12]: [192, 184, 190, 188, 180, 193, 193, 192, 188, 189, 189, 192]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.711777873Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255152023-09-22T23:36:00.774Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.711855869Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255162023-09-22T23:36:00.777Z{"msg":"[2]R flush_numbers[0..12]: [322, 308, 317, 315, 291, 323, 323, 320, 314, 316, 316, 321]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.711914697Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255172023-09-22T23:36:00.781Z{"msg":"[2]R generation[0..12]: [192, 184, 190, 188, 180, 193, 193, 192, 188, 189, 189, 192]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.711978521Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255182023-09-22T23:36:00.783Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.712037471Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255192023-09-22T23:36:00.786Z{"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.71209952Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255202023-09-22T23:36:00.790Z{"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.712164866Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255212023-09-22T23:36:00.792Z{"msg":"Next flush: 324","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.712220515Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255222023-09-22T23:36:00.795Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.712274346Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255232023-09-22T23:36:00.798Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.712329658Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255242023-09-22T23:36:00.801Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.71239358Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255252023-09-22T23:36:00.804Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.712453586Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255262023-09-22T23:36:00.807Z{"msg":"7a8c4a1e-9bb9-4923-837a-7438fb102f98 is now active with session: 0d46d172-b8c5-4227-a373-9ee85552cc9b","v":0,"name":"crucible","level":30Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
255272023-09-22T23:36:00.810Z,"time":"2023-09-22T23:36:00.712522154Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255282023-09-22T23:36:00.813Z{"msg":"7a8c4a1e-9bb9-4923-837a-7438fb102f98 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.712644151Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255292023-09-22T23:36:00.815Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.712701739Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255302023-09-22T23:36:00.818Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.712756131Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255312023-09-22T23:36:00.821Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.712809753Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255322023-09-22T23:36:00.824Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.712863459Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255332023-09-22T23:36:00.826Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.712916257Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255342023-09-22T23:36:00.829Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.71296955Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255352023-09-22T23:36:00.832Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.713023289Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255362023-09-22T23:36:00.835Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.713076772Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255372023-09-22T23:36:00.840Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.713130386Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255382023-09-22T23:36:00.843ZLoading write count information from file "/tmp/test_repair_verify.data"
255392023-09-22T23:36:00.845Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30Run Repair workload
255402023-09-22T23:36:00.851Z,"time":"2023-09-22T23:36:00.713188101Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255412023-09-22T23:36:00.854Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.713297466Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1959}
255422023-09-22T23:36:00.857Z01/30 Write block 559 len 40960 data: 14 16 14 14 14 16 16 16 18 18
255432023-09-22T23:36:00.859Z02/30 Write block 54 len 24576 data: 8 9 11 14 16 16
255442023-09-22T23:36:00.862Z03/30 Write block 157 len 8192 data: 15 17
255452023-09-22T23:36:00.865Z04/30 Write block 249 len 24576 data: 13 12 11 15 13 12
255462023-09-22T23:36:00.868Z05/30 Write block 330 len 28672 data: 12 15 16 15 20 20 19
255472023-09-22T23:36:00.870Z06/30 Read block 262 len 32768
255482023-09-22T23:36:00.873Z07/30 Write block 354 len 32768 data: 17 18 17 14 17 20 19 15
255492023-09-22T23:36:00.876Z08/30 Read block 527 len 12288
255502023-09-22T23:36:00.879Z09/30 Write block 435 len 36864 data: 19 18 17 19 17 14 16 18 21
255512023-09-22T23:36:00.882Z10/30 Write block 496 len 28672 data: 21 22 15 15 14 15 16
255522023-09-22T23:36:00.885Z11/30 Write block 52 len 24576 data: 11 9 9 10 12 15
255532023-09-22T23:36:00.888Z12/30 Flush
255542023-09-22T23:36:00.891Z13/30 Write block 459 len 16384 data: 19 19 18 17
255552023-09-22T23:36:00.894Z14/30 Flush
255562023-09-22T23:36:00.897Z15/30 Write block 135 len 8192 data: 11 12
255572023-09-22T23:36:00.899Z16/30 Read block 202 len 28672
255582023-09-22T23:36:00.902Z17/30 Read block 38 len 36864
255592023-09-22T23:36:00.905Z18/30 Write block 541 len 16384 data: 10 12 13 10
255602023-09-22T23:36:00.909Z19/30 Read block 564 len 40960
255612023-09-22T23:36:00.912Z20/30 Read block 282 len 4096
255622023-09-22T23:36:00.915Z21/30 Read block 268 len 16384
255632023-09-22T23:36:00.918Z22/30 Write block 239 len 16384 data: 16 19 19 16
255642023-09-22T23:36:00.921Z23/30 Write block 493 len 4096 data: 21
255652023-09-22T23:36:00.924Z24/30 Write block 425 len 4096 data: 10
255662023-09-22T23:36:00.927Z25/30 Read block 98 len 36864
255672023-09-22T23:36:00.930Z26/30 Read block 172 len 28672
255682023-09-22T23:36:00.933Z27/30 Read block 46 len 32768
255692023-09-22T23:36:00.936Z28/30 Read block 522 len 40960
255702023-09-22T23:36:00.939Z29/30 Read block 501 len 20480
255712023-09-22T23:36:00.942Z30/30 Read block 501 len 20480
255722023-09-22T23:36:00.945Z----------------------------------------------------------------
255732023-09-22T23:36:00.948Z Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30
255742023-09-22T23:36:00.951ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
255752023-09-22T23:36:00.954Z 1 Acked 1000 Write 10 Done Done Done false
255762023-09-22T23:36:00.957Z 2 Acked 1001 Write 6 Done Done Done false
255772023-09-22T23:36:00.960Z 3 Acked 1002 Write 2 Done Done Done false
255782023-09-22T23:36:00.963Z 4 Acked 1003 Write 6 Done Done Done false
255792023-09-22T23:36:00.965Z 5 Acked 1004 Write 7 Done Done Done false
255802023-09-22T23:36:00.968Z 6 Acked 1005 Read 8 Done Done Done false
255812023-09-22T23:36:00.971Z 7 Acked 1006 Write 8 Sent Done Done false
255822023-09-22T23:36:00.974Z 8 Acked 1007 Read 3 Sent Done Done false
255832023-09-22T23:36:00.976Z 9 Acked 1008 Write 9 Sent Done Done false
255842023-09-22T23:36:00.979Z 10 Acked 1009 Write 7 Sent Done Done false
255852023-09-22T23:36:00.982Z 11 Acked 1010 Write 6 Sent Done Done false
255862023-09-22T23:36:00.985Z 12 Acked 1011 Flush 0 Sent Done Done false
255872023-09-22T23:36:00.988Z 13 Acked 1012 Write 4 Sent Done Done false
255882023-09-22T23:36:00.991Z 14 Acked 1013 Flush 0 Sent Done Done false
255892023-09-22T23:36:00.994Z 15 Acked 1014 Write 2 Sent Done Done false
255902023-09-22T23:36:00.997Z 16 Acked 1015 Read 7 Sent Done Done false
255912023-09-22T23:36:01.000Z 17 Acked 1016 Read 9 Sent Done Done false
255922023-09-22T23:36:01.004Z 18 Acked 1017 Write 4 Sent Done Done false
255932023-09-22T23:36:01.007Z 19 Acked 1018 Read 10 Sent Done Done false
255942023-09-22T23:36:01.010Z 20 Acked 1019 Read 1 Sent Done Done false
255952023-09-22T23:36:01.013Z 21 Acked 1020 Read 4 Sent Done Done false
255962023-09-22T23:36:01.016Z 22 Acked 1021 Write 4 Sent Done Done false
255972023-09-22T23:36:01.019Z 23 Acked 1022 Write 1 Sent Done Done false
255982023-09-22T23:36:01.021Z 24 Acked 1023 Write 1 Sent Done Done false
255992023-09-22T23:36:01.024Z 25 Acked 1024 Read 9 Sent Done Done false
256002023-09-22T23:36:01.027Z 26 Acked 1025 Read 7 Sent Done Done false
256012023-09-22T23:36:01.030Z 27 Acked 1026 Read 8 Sent Done Done false
256022023-09-22T23:36:01.033Z 28 Acked 1027 Read 10 Sent Done Done false
256032023-09-22T23:36:01.036Z 29 Acked 1028 Read 5 Sent Done Done false
256042023-09-22T23:36:01.038Z 30 Acked 1029 Read 5 Sent Sent Done false
256052023-09-22T23:36:01.041Z STATES DS:0 DS:1 DS:2 TOTAL
256062023-09-22T23:36:01.045Z New 0 0 0 0
256072023-09-22T23:36:01.048Z Sent 24 1 0 25
256082023-09-22T23:36:01.051Z Done 6 29 30 65
256092023-09-22T23:36:01.054Z Skipped 0 0 0 0
256102023-09-22T23:36:01.058Z Error 0 0 0 0
256112023-09-22T23:36:01.061ZLast Flush: 0 1013 1013
256122023-09-22T23:36:01.064ZDownstairs last five completed:
256132023-09-22T23:36:01.066ZUpstairs last five completed: 30 29 28 27 26 25
256142023-09-22T23:36:01.069ZWrote out file "/tmp/test_repair_verify.data"
256152023-09-22T23:36:01.073Z
256162023-09-22T23:36:01.076ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
256172023-09-22T23:36:01.079Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.865681053Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1960}
256182023-09-22T23:36:01.081Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.865992734Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1960}
256192023-09-22T23:36:01.085Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.866016348Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1960}
256202023-09-22T23:36:01.088Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.914483449Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1960}
256212023-09-22T23:36:01.090Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.914547501Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1960}
256222023-09-22T23:36:01.093Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.914587035Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1960}
256232023-09-22T23:36:01.096Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.934297966Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1960}
256242023-09-22T23:36:01.099Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.934361978Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1960}
256252023-09-22T23:36:01.102Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:00.934418388Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1960}
256262023-09-22T23:36:01.105ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
256272023-09-22T23:36:01.108Z 0 000-019 192 192 192 322 322 322 F F F
256282023-09-22T23:36:01.111Z 1 020-039 184 184 184 308 308 308 F F F
256292023-09-22T23:36:01.115Z 2 040-059 190 194 194 317 324 324 T F F <---
256302023-09-22T23:36:01.118Z 3 060-079 188 188 188 315 315 315 F F F
256312023-09-22T23:36:01.121Z 4 080-099 180 180 180 291 291 291 F F F
256322023-09-22T23:36:01.124Z 5 100-119 193 193 193 323 323 323 F F F
256332023-09-22T23:36:01.127Z 6 120-139 193 193 193 323 323 323 F T T <---
256342023-09-22T23:36:01.130Z 7 140-159 192 194 194 320 324 324 T F F <---
256352023-09-22T23:36:01.132Z 8 160-179 188 188 188 314 314 314 F F F
256362023-09-22T23:36:01.135Z 9 180-199 189 189 189 316 316 316 F F F
256372023-09-22T23:36:01.138Z 10 200-219 189 189 189 316 316 316 F F F
256382023-09-22T23:36:01.141Z 11 220-239 192 192 192 321 321 321 F T T <---
256392023-09-22T23:36:01.144Z 12 240-259 188 194 194 315 324 324 T T T <---
256402023-09-22T23:36:01.147Z 13 260-279 192 192 192 321 321 321 F F F
256412023-09-22T23:36:01.149ZError: Difference in extent metadata found!
256422023-09-22T23:36:01.152Z 14 280-299 184 184 184 305 305 305 F F F
256432023-09-22T23:36:01.155Z 15 300-319 192 192 192 320 320 320 F F F
256442023-09-22T23:36:01.158Z 16 320-339 191 194 194 319 324 324 T F F <---
256452023-09-22T23:36:01.161Z 17 340-359 191 194 194 319 324 324 F F F <---
256462023-09-22T23:36:01.164Z 18 360-379 189 194 194 316 324 324 F F F <---
256472023-09-22T23:36:01.166Z 19 380-399 188 188 188 315 315 315 F F F
256482023-09-22T23:36:01.170Z 20 400-419 193 193 193 323 323 323 F F F
256492023-09-22T23:36:01.173Z 21 420-439 192 194 194 320 324 324 F T T <---
256502023-09-22T23:36:01.175Z 22 440-459 191 194 194 319 325 325 F F F <---
256512023-09-22T23:36:01.178Z 23 460-479 193 194 194 323 325 325 F F F <---
256522023-09-22T23:36:01.181Z 24 480-499 191 194 194 319 324 324 F T T <---
256532023-09-22T23:36:01.184Z 25 500-519 192 194 194 321 324 324 F F F <---
256542023-09-22T23:36:01.187Z 26 520-539 193 193 193 323 323 323 F F F
256552023-09-22T23:36:01.189Z 27 540-559 191 194 194 319 324 324 T T T <---
256562023-09-22T23:36:01.192Z 28 560-579 191 194 194 319 324 324 T F F <---
256572023-09-22T23:36:01.195Z 29 580-599 192 192 192 320 320 320 F F F
256582023-09-22T23:36:01.199ZMax gen: 194, Max flush: 325
256592023-09-22T23:36:01.202ZA Difference in extent metadata is expected here
256602023-09-22T23:36:01.205ZOn loop 97
256612023-09-22T23:36:01.208Z
256622023-09-22T23:36:01.211ZVerifying data now
256632023-09-22T23:36:07.543ZNo change after verify
256642023-09-22T23:36:07.546ZLoop: 97 Downstairs dump after verify (and repair):
256652023-09-22T23:36:07.561Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.559389098Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1965}
256662023-09-22T23:36:07.564Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.559749482Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1965}
256672023-09-22T23:36:07.566Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.5597749Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1965}
256682023-09-22T23:36:07.580Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.578548355Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1965}
256692023-09-22T23:36:07.583Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.578602031Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1965}
256702023-09-22T23:36:07.586Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.578622853Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1965}
256712023-09-22T23:36:07.591Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.589670488Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1965}
256722023-09-22T23:36:07.594Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.589722806Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1965}
256732023-09-22T23:36:07.597Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.589743136Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1965}
256742023-09-22T23:36:07.603ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
256752023-09-22T23:36:07.606Z 0 000-019 192 192 192 322 322 322 F F F
256762023-09-22T23:36:07.608Z 1 020-039 184 184 184 308 308 308 F F F
256772023-09-22T23:36:07.611Z 2 040-059 194 194 194 324 324 324 F F F
256782023-09-22T23:36:07.615Z 3 060-079 188 188 188 315 315 315 F F F
256792023-09-22T23:36:07.617Z 4 080-099 180 180 180 291 291 291 F F F
256802023-09-22T23:36:07.620Z 5 100-119 193 193 193 323 323 323 F F F
256812023-09-22T23:36:07.623Z 6 120-139 195 195 195 326 326 326 F F F
256822023-09-22T23:36:07.626Z 7 140-159 194 194 194 324 324 324 F F F
256832023-09-22T23:36:07.629Z 8 160-179 188 188 188 314 314 314 F F F
256842023-09-22T23:36:07.631Z 9 180-199 189 189 189 316 316 316 F F F
256852023-09-22T23:36:07.634Z 10 200-219 189 189 189 316 316 316 F F F
256862023-09-22T23:36:07.637Z 11 220-239 195 195 195 326 326 326 F F F
256872023-09-22T23:36:07.642Z 12 240-259 195 195 195 326 326 326 F F F
256882023-09-22T23:36:07.646Z 13 260-279 192 192 192 321 321 321 F F F
256892023-09-22T23:36:07.649Z 14 280-299 184 184 184 305 305 305 F F F
256902023-09-22T23:36:07.652Z 15 300-319 192 192 192 320 320 320 F F F
256912023-09-22T23:36:07.655Z 16 320-339 194 194 194 324 324 324 F F F
256922023-09-22T23:36:07.657Z 17 340-359 194 194 194 324 324 324 F F F
256932023-09-22T23:36:07.660Z 18 360-379 194 194 194 324 324 324 F F F
256942023-09-22T23:36:07.663Z 19 380-399 188 188 188 315 315 315 F F F
256952023-09-22T23:36:07.666Z 20 400-419 193 193 193 323 323 323 F F F
256962023-09-22T23:36:07.669Z 21 420-439 195 195 195 326 326 326 F F F
256972023-09-22T23:36:07.672Z 22 440-459 194 194 194 325 325 325 F F F
256982023-09-22T23:36:07.674Z 23 460-479 194 194 194 325 325 325 F F F
256992023-09-22T23:36:07.679Z 24 480-499 195 195 195 326 326 326 F F F
257002023-09-22T23:36:07.682Z 25 500-519 194 194 194 324 324 324 F F F
257012023-09-22T23:36:07.684Z 26 520-539 193 193 193 323 323 323 F F F
257022023-09-22T23:36:07.687Z 27 540-559 195 195 195 326 326 326 F F F
257032023-09-22T23:36:07.689Z 28 560-579 194 194 194 324 324 324 F F F
257042023-09-22T23:36:07.692Z 29 580-599 192 192 192 320 320 320 F F F
257052023-09-22T23:36:07.695ZMax gen: 195, Max flush: 326
257062023-09-22T23:36:07.698Z
257072023-09-22T23:36:07.701ZBegin loop 98
257082023-09-22T23:36:07.704ZDownstairs to restart: 1
257092023-09-22T23:36:07.706Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.641578866Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257102023-09-22T23:36:07.709Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.641954807Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257112023-09-22T23:36:07.712Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.642029967Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257122023-09-22T23:36:07.715Z{"msg":"Crucible stats registered with UUID: fd814412-012d-4a8d-98bc-77ca758ba232","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.642057042Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257132023-09-22T23:36:07.718Z{"msg":"Crucible fd814412-012d-4a8d-98bc-77ca758ba232 has session id: dd012528-808f-479a-9641-fa7959676e52","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.642114876Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257142023-09-22T23:36:07.720ZCrucible runtime is spawned
257152023-09-22T23:36:07.723ZThe guest has requested activation with gen:196
257162023-09-22T23:36:07.726Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.642442009Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967,"looper":"2"}
257172023-09-22T23:36:07.728Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.642526824Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967,"task":"up_listen"}
257182023-09-22T23:36:07.731Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.642583684Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257192023-09-22T23:36:07.734Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.642623412Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257202023-09-22T23:36:07.737Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.642781296Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967,"looper":"0"}
257212023-09-22T23:36:07.740Z{"msg":"[2] fd814412-012d-4a8d-98bc-77ca758ba232 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.642927291Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967,"looper":"2"}
257222023-09-22T23:36:07.742Z{"msg":"[0] fd814412-012d-4a8d-98bc-77ca758ba232 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.643032188Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967,"looper":"0"}
257232023-09-22T23:36:07.745Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.643155412Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967,"looper":"1"}
257242023-09-22T23:36:07.748Z{"msg":"fd814412-012d-4a8d-98bc-77ca758ba232 active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.643608983Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257252023-09-22T23:36:07.751Z{"msg":"Set desired generation to :196","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.643675528Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257262023-09-22T23:36:07.754Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.643800046Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257272023-09-22T23:36:07.756Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.644101398Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257282023-09-22T23:36:07.758Z{"msg":"[2] fd814412-012d-4a8d-98bc-77ca758ba232 (a82ea716-51df-45b6-8f3d-10405ff7faff) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.64450398Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257292023-09-22T23:36:07.761Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.644575441Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257302023-09-22T23:36:07.764Z{"msg":"[2] client is_active_req TRUE, promote! session a82ea716-51df-45b6-8f3d-10405ff7faff","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.644635844Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257312023-09-22T23:36:07.767Z{"msg":"[0] fd814412-012d-4a8d-98bc-77ca758ba232 (a82ea716-51df-45b6-8f3d-10405ff7faff) New New WaitActive ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.644704282Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257322023-09-22T23:36:07.769Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.644740516Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257332023-09-22T23:36:07.772Z{"msg":"[0] client is_active_req TRUE, promote! session a82ea716-51df-45b6-8f3d-10405ff7faff","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.644774236Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257342023-09-22T23:36:07.775Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.645241678Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257352023-09-22T23:36:07.778Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.645315892Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257362023-09-22T23:36:07.781Z{"msg":"fd814412-012d-4a8d-98bc-77ca758ba232 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.64537076Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257372023-09-22T23:36:07.783Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.6454382Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257382023-09-22T23:36:07.787Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.645513765Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257392023-09-22T23:36:07.790Z{"msg":"fd814412-012d-4a8d-98bc-77ca758ba232 WaitActive New WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.645567241Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257402023-09-22T23:36:07.793Z{"msg":"[2] fd814412-012d-4a8d-98bc-77ca758ba232 (a82ea716-51df-45b6-8f3d-10405ff7faff) WaitActive New WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.647690227Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257412023-09-22T23:36:07.797Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.64778122Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257422023-09-22T23:36:07.800Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:36:07.647840256Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257432023-09-22T23:36:07.802Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.647882812Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257442023-09-22T23:36:07.805Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.647922966Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257452023-09-22T23:36:07.808Z{"msg":"fd814412-012d-4a8d-98bc-77ca758ba232 WaitActive New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.647963073Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257462023-09-22T23:36:07.811Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.648002129Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257472023-09-22T23:36:07.814Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.648050317Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257482023-09-22T23:36:07.819Z{"msg":"[0] fd814412-012d-4a8d-98bc-77ca758ba232 (a82ea716-51df-45b6-8f3d-10405ff7faff) WaitActive New WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.648895248Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257492023-09-22T23:36:07.822Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.648975404Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257502023-09-22T23:36:07.825Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:36:07.649020832Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257512023-09-22T23:36:07.827Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.649061815Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257522023-09-22T23:36:07.830Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.649101526Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257532023-09-22T23:36:07.833Z{"msg":"fd814412-012d-4a8d-98bc-77ca758ba232 WaitQuorum New WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.649140669Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257542023-09-22T23:36:07.837Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.649179166Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257552023-09-22T23:36:07.840Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:07.649217056Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257562023-09-22T23:36:08.646Z{"msg":"[1] fd814412-012d-4a8d-98bc-77ca758ba232 looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.644039195Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967,"looper":"1"}
257572023-09-22T23:36:08.649Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.644123451Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257582023-09-22T23:36:08.652Z{"msg":"[1] fd814412-012d-4a8d-98bc-77ca758ba232 (a82ea716-51df-45b6-8f3d-10405ff7faff) WaitQuorum New WaitQuorum ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.645454385Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257592023-09-22T23:36:08.655Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.645510852Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257602023-09-22T23:36:08.658Z{"msg":"[1] client is_active_req TRUE, promote! session a82ea716-51df-45b6-8f3d-10405ff7faff","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.645554736Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257612023-09-22T23:36:08.660Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.646138368Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257622023-09-22T23:36:08.663Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.646198926Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257632023-09-22T23:36:08.666Z{"msg":"fd814412-012d-4a8d-98bc-77ca758ba232 WaitQuorum WaitActive WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.646241006Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257642023-09-22T23:36:08.669Z{"msg":"[1] fd814412-012d-4a8d-98bc-77ca758ba232 (a82ea716-51df-45b6-8f3d-10405ff7faff) WaitQuorum WaitActive WaitQuorum ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.648640159Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257652023-09-22T23:36:08.672Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.648695165Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257662023-09-22T23:36:08.675Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:36:08.648741426Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257672023-09-22T23:36:08.678Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.6487805Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257682023-09-22T23:36:08.681Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.648820966Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257692023-09-22T23:36:08.684Z{"msg":"fd814412-012d-4a8d-98bc-77ca758ba232 WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.648860996Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257702023-09-22T23:36:08.687Z{"msg":"[0]R flush_numbers[0..12]: [322, 308, 324, 315, 291, 323, 326, 324, 314, 316, 316, 326]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.648902026Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257712023-09-22T23:36:08.689Z{"msg":"[0]R generation[0..12]: [192, 184, 194, 188, 180, 193, 195, 194, 188, 189, 189, 195]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.648942697Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257722023-09-22T23:36:08.692Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.648984203Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257732023-09-22T23:36:08.695Z{"msg":"[1]R flush_numbers[0..12]: [322, 308, 324, 315, 291, 323, 326, 324, 314, 316, 316, 326]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.64902442Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257742023-09-22T23:36:08.698Z{"msg":"[1]R generation[0..12]: [192, 184, 194, 188, 180, 193, 195, 194, 188, 189, 189, 195]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.649066431Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257752023-09-22T23:36:08.700ZThe guest has finished waiting for activation with:196
257762023-09-22T23:36:08.703Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","Wait for a query_work_queue command to finish before sending IO
257772023-09-22T23:36:08.706Zv":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.649123205Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257782023-09-22T23:36:08.709Z{"msg":"[2]R flush_numbers[0..12]: [322, 308, 324, 315, 291, 323, 326, 324, 314, 316, 316, 326]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.649173551Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257792023-09-22T23:36:08.711Z{"msg":"[2]R generation[0..12]: [192, 184, 194, 188, 180, 193, 195, 194, 188, 189, 189, 195]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.649240978Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257802023-09-22T23:36:08.715Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.649347056Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257812023-09-22T23:36:08.718Z{"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.649420425Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257822023-09-22T23:36:08.727Z{"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.649465719Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257832023-09-22T23:36:08.733Z{"msg":"Next flush: 327","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.649513994Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257842023-09-22T23:36:08.736Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.649571616Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257852023-09-22T23:36:08.738Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.649642899Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257862023-09-22T23:36:08.741Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.649688937Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257872023-09-22T23:36:08.744Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.649735224Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257882023-09-22T23:36:08.747Z{"msg":"fd814412-012d-4a8d-98bc-77ca758ba232 is now active with session: a82ea716-51df-45b6-8f3d-10405ff7faff","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.6497934Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257892023-09-22T23:36:08.749Z{"msg":"fd814412-012d-4a8d-98bc-77ca758ba232 Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.649840035Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257902023-09-22T23:36:08.752Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.649883868Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257912023-09-22T23:36:08.755Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.649904841Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257922023-09-22T23:36:08.758Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.649958092Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257932023-09-22T23:36:08.761Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.650024791Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257942023-09-22T23:36:08.763Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.650066254Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257952023-09-22T23:36:08.766Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.650087239Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257962023-09-22T23:36:08.769Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.650145054Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257972023-09-22T23:36:08.772Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucibleRegion: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
257982023-09-22T23:36:08.775Z","level":30,"time":"2023-09-22T23:36:08.650197423Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
257992023-09-22T23:36:08.778Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.650236742Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
258002023-09-22T23:36:08.781Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.650275013Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
258012023-09-22T23:36:08.784Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.650295424Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1967}
258022023-09-22T23:36:08.787ZLoading write count information from file "/tmp/test_repair_verify.data"
258032023-09-22T23:36:08.789ZRun Repair workload
258042023-09-22T23:36:08.792Z01/30 Write block 575 len 40960 data: 11 12 13 13 10 12 12 13 12 11
258052023-09-22T23:36:08.795Z02/30 Flush
258062023-09-22T23:36:08.798Z03/30 Write block 425 len 20480 data: 11 14 16 16 15
258072023-09-22T23:36:08.801Z04/30 Read block 158 len 36864
258082023-09-22T23:36:08.804Z05/30 Write block 415 len 36864 data: 12 12 13 18 20 18 17 14 15
258092023-09-22T23:36:08.807Z06/30 Write block 406 len 8192 data: 12 13
258102023-09-22T23:36:08.809Z07/30 Read block 186 len 16384
258112023-09-22T23:36:08.812Z08/30 Write block 77 len 12288 data: 18 18 21
258122023-09-22T23:36:08.815Z09/30 Write block 200 len 16384 data: 11 14 17 20
258132023-09-22T23:36:08.817Z10/30 Write block 122 len 36864 data: 14 16 14 11 9 13 13 11 12
258142023-09-22T23:36:08.820Z11/30 Read block 481 len 24576
258152023-09-22T23:36:08.823Z12/30 Read block 90 len 24576
258162023-09-22T23:36:08.832Z13/30 Write block 582 len 36864 data: 14 13 12 13 11 12 13 13 13
258172023-09-22T23:36:08.835Z14/30 Write block 108 len 32768 data: 24 23 24 21 21 20 19 15
258182023-09-22T23:36:08.838Z15/30 Read block 511 len 20480
258192023-09-22T23:36:08.841Z16/30 Read block 122 len 32768
258202023-09-22T23:36:08.844Z17/30 Write block 177 len 8192 data: 21 19
258212023-09-22T23:36:08.847Z18/30 Read block 545 len 32768
258222023-09-22T23:36:08.849Z19/30 Write block 370 len 20480 data: 11 13 13 10 12
258232023-09-22T23:36:08.852Z20/30 Flush
258242023-09-22T23:36:08.855Z21/30 Write block 171 len 4096 data: 11
258252023-09-22T23:36:08.858Z22/30 Flush
258262023-09-22T23:36:08.861Z23/30 Write block 214 len 20480 data: 12 13 12 12 11
258272023-09-22T23:36:08.863Z24/30 Write block 18 len 40960 data: 14 13 10 8 10 8 11 12 13 15
258282023-09-22T23:36:08.866Z25/30 Write block 178 len 12288 data: 20 16 13
258292023-09-22T23:36:08.869Z26/30 Read block 118 len 12288
258302023-09-22T23:36:08.872Z27/30 Read block 438 len 36864
258312023-09-22T23:36:08.874Z28/30 Read block 157 len 16384
258322023-09-22T23:36:08.877Z29/30 Read block 459 len 40960
258332023-09-22T23:36:08.880Z30/30 Write block 414 len 40960 data: 12 13 13 14 19 21 19 18 15 16
258342023-09-22T23:36:08.883Z----------------------------------------------------------------
258352023-09-22T23:36:08.886Z Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:28
258362023-09-22T23:36:08.889ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
258372023-09-22T23:36:08.892Z 3 Acked 1002 Write 5 Done Sent Done false
258382023-09-22T23:36:08.895Z 4 Acked 1003 Read 9 Done Sent Done false
258392023-09-22T23:36:08.898Z 5 Acked 1004 Write 9 Done Sent Done false
258402023-09-22T23:36:08.900Z 6 Acked 1005 Write 2 Done Sent Done false
258412023-09-22T23:36:08.904Z 7 Acked 1006 Read 4 Done Sent Done false
258422023-09-22T23:36:08.907Z 8 Acked 1007 Write 3 Done Sent Done false
258432023-09-22T23:36:08.909Z 9 Acked 1008 Write 4 Done Sent Done false
258442023-09-22T23:36:08.912Z 10 Acked 1009 Write 9 Done Sent Done false
258452023-09-22T23:36:08.916Z 11 Acked 1010 Read 6 Done Sent Done false
258462023-09-22T23:36:08.919Z 12 Acked 1011 Read 6 Done Sent Done false
258472023-09-22T23:36:08.922Z 13 Acked 1012 Write 9 Done Sent Done false
258482023-09-22T23:36:08.925Z 14 Acked 1013 Write 8 Done Sent Done false
258492023-09-22T23:36:08.927Z 15 Acked 1014 Read 5 Done Sent Done false
258502023-09-22T23:36:08.930Z 16 Acked 1015 Read 8 Done Sent Done false
258512023-09-22T23:36:08.933Z 17 Acked 1016 Write 2 Done Sent Done false
258522023-09-22T23:36:08.936Z 18 Acked 1017 Read 8 Done Sent Done false
258532023-09-22T23:36:08.938Z 19 Acked 1018 Write 5 Done Sent Done false
258542023-09-22T23:36:08.942Z 20 Acked 1019 Flush 0 Done Sent Done false
258552023-09-22T23:36:08.945Z 21 Acked 1020 Write 1 Done Sent Done false
258562023-09-22T23:36:08.947Z 22 Acked 1021 Flush 0 Done Sent Done false
258572023-09-22T23:36:08.950Z 23 Acked 1022 Write 5 Done Sent Done false
258582023-09-22T23:36:08.953Z 24 Acked 1023 Write 10 Done Sent Done false
258592023-09-22T23:36:08.956Z 25 Acked 1024 Write 3 Done Sent Done false
258602023-09-22T23:36:08.959Z 26 Acked 1025 Read 3 Done Sent Done false
258612023-09-22T23:36:08.961Z 27 Acked 1026 Read 9 Done Sent Done false
258622023-09-22T23:36:08.965Z 28 Acked 1027 Read 4 Done Sent Done false
258632023-09-22T23:36:08.968Z 29 Acked 1028 Read 10 Done Sent Done false
258642023-09-22T23:36:08.971Z 30 Acked 1029 Write 10 Sent Sent Sent false
258652023-09-22T23:36:08.974Z STATES DS:0 DS:1 DS:2 TOTAL
258662023-09-22T23:36:08.976Z New 0 0 0 0
258672023-09-22T23:36:08.980Z Sent 1 28 1 30
258682023-09-22T23:36:08.983Z Done 27 0 27 54
258692023-09-22T23:36:08.985Z Skipped 0 0 0 0
258702023-09-22T23:36:08.988Z Error 0 0 0 0
258712023-09-22T23:36:08.991ZLast Flush: 1021 1001 1021
258722023-09-22T23:36:08.994ZDownstairs last five completed: 1001 1000
258732023-09-22T23:36:08.996ZUpstairs last five completed: 30 29 28 27 26 25
258742023-09-22T23:36:08.999ZWrote out file "/tmp/test_repair_verify.data"
258752023-09-22T23:36:09.002Z
258762023-09-22T23:36:09.005ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
258772023-09-22T23:36:09.008Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.799085042Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1968}
258782023-09-22T23:36:09.010Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.799411913Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1968}
258792023-09-22T23:36:09.013Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.799435259Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1968}
258802023-09-22T23:36:09.016Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.828874257Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1968}
258812023-09-22T23:36:09.019Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.828985146Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1968}
258822023-09-22T23:36:09.021Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.829033661Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1968}
258832023-09-22T23:36:09.024Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.847968626Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1968}
258842023-09-22T23:36:09.027Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.848021875Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1968}
258852023-09-22T23:36:09.030Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:08.848060847Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1968}
258862023-09-22T23:36:09.033ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
258872023-09-22T23:36:09.035Z 0 000-019 192 192 192 322 322 322 T F T <---
258882023-09-22T23:36:09.038Z 1 020-039 184 184 184 308 308 308 T F T <---
258892023-09-22T23:36:09.041Z 2 040-059 194 194 194 324 324 324 F F F
258902023-09-22T23:36:09.044Z 3 060-079 196 188 196 328 315 328 F F F <---
258912023-09-22T23:36:09.047Z 4 080-099 180 180 180 291 291 291 F F F
258922023-09-22T23:36:09.050Z 5 100-119 196 193 196 328 323 328 F F F <---
258932023-09-22T23:36:09.054Z 6 120-139 196 195 196 328 326 328 F F F <---
258942023-09-22T23:36:09.057Z 7 140-159 194 194 194 324 324 324 F F F
258952023-09-22T23:36:09.060ZError: Difference in extent metadata found!
258962023-09-22T23:36:09.062Z 8 160-179 196 188 196 329 314 329 T F T <---
258972023-09-22T23:36:09.065Z 9 180-199 189 189 189 316 316 316 T F T <---
258982023-09-22T23:36:09.068Z 10 200-219 196 189 196 328 316 328 T F T <---
258992023-09-22T23:36:09.071Z 11 220-239 195 195 195 326 326 326 F F F
259002023-09-22T23:36:09.076Z 12 240-259 195 195 195 326 326 326 F F F
259012023-09-22T23:36:09.079Z 13 260-279 192 192 192 321 321 321 F F F
259022023-09-22T23:36:09.081Z 14 280-299 184 184 184 305 305 305 F F F
259032023-09-22T23:36:09.084Z 15 300-319 192 192 192 320 320 320 F F F
259042023-09-22T23:36:09.087Z 16 320-339 194 194 194 324 324 324 F F F
259052023-09-22T23:36:09.090Z 17 340-359 194 194 194 324 324 324 F F F
259062023-09-22T23:36:09.092Z 18 360-379 196 194 196 328 324 328 F F F <---
259072023-09-22T23:36:09.095Z 19 380-399 188 188 188 315 315 315 F F F
259082023-09-22T23:36:09.098Z 20 400-419 196 193 196 328 323 328 T F T <---
259092023-09-22T23:36:09.101Z 21 420-439 196 195 196 328 326 328 T F T <---
259102023-09-22T23:36:09.104Z 22 440-459 194 194 194 325 325 325 F F F
259112023-09-22T23:36:09.107Z 23 460-479 194 194 194 325 325 325 F F F
259122023-09-22T23:36:09.110Z 24 480-499 195 195 195 326 326 326 F F F
259132023-09-22T23:36:09.112Z 25 500-519 194 194 194 324 324 324 F F F
259142023-09-22T23:36:09.115Z 26 520-539 193 193 193 323 323 323 F F F
259152023-09-22T23:36:09.118Z 27 540-559 195 195 195 326 326 326 F F F
259162023-09-22T23:36:09.121Z 28 560-579 196 196 196 327 327 327 F F F
259172023-09-22T23:36:09.124Z 29 580-599 196 196 196 328 327 328 F F F <---
259182023-09-22T23:36:09.127ZMax gen: 196, Max flush: 329
259192023-09-22T23:36:09.129ZA Difference in extent metadata is expected here
259202023-09-22T23:36:09.132ZOn loop 98
259212023-09-22T23:36:09.135Z
259222023-09-22T23:36:09.137ZVerifying data now
259232023-09-22T23:36:15.200ZNo change after verify
259242023-09-22T23:36:15.205ZLoop: 98 Downstairs dump after verify (and repair):
259252023-09-22T23:36:15.221Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.217152461Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1973}
259262023-09-22T23:36:15.224Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.217580546Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1973}
259272023-09-22T23:36:15.227Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.217605557Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1973}
259282023-09-22T23:36:15.237Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.236033862Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1973}
259292023-09-22T23:36:15.244Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.23608656Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1973}
259302023-09-22T23:36:15.249Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.236107299Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1973}
259312023-09-22T23:36:15.252Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.246978903Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1973}
259322023-09-22T23:36:15.255Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.247030592Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1973}
259332023-09-22T23:36:15.257Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.247068358Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1973}
259342023-09-22T23:36:15.260ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
259352023-09-22T23:36:15.266Z 0 000-019 197 197 197 330 330 330 F F F
259362023-09-22T23:36:15.269Z 1 020-039 197 197 197 330 330 330 F F F
259372023-09-22T23:36:15.274Z 2 040-059 194 194 194 324 324 324 F F F
259382023-09-22T23:36:15.279Z 3 060-079 196 196 196 328 328 328 F F F
259392023-09-22T23:36:15.283Z 4 080-099 180 180 180 291 291 291 F F F
259402023-09-22T23:36:15.286Z 5 100-119 196 196 196 328 328 328 F F F
259412023-09-22T23:36:15.291Z 6 120-139 196 196 196 328 328 328 F F F
259422023-09-22T23:36:15.294Z 7 140-159 194 194 194 324 324 324 F F F
259432023-09-22T23:36:15.298Z 8 160-179 197 197 197 330 330 330 F F F
259442023-09-22T23:36:15.301Z 9 180-199 197 197 197 330 330 330 F F F
259452023-09-22T23:36:15.304Z 10 200-219 197 197 197 330 330 330 F F F
259462023-09-22T23:36:15.306Z 11 220-239 195 195 195 326 326 326 F F F
259472023-09-22T23:36:15.309Z 12 240-259 195 195 195 326 326 326 F F F
259482023-09-22T23:36:15.312Z 13 260-279 192 192 192 321 321 321 F F F
259492023-09-22T23:36:15.315Z 14 280-299 184 184 184 305 305 305 F F F
259502023-09-22T23:36:15.318Z 15 300-319 192 192 192 320 320 320 F F F
259512023-09-22T23:36:15.321Z 16 320-339 194 194 194 324 324 324 F F F
259522023-09-22T23:36:15.324Z 17 340-359 194 194 194 324 324 324 F F F
259532023-09-22T23:36:15.327Z 18 360-379 196 196 196 328 328 328 F F F
259542023-09-22T23:36:15.330Z 19 380-399 188 188 188 315 315 315 F F F
259552023-09-22T23:36:15.334Z 20 400-419 197 197 197 330 330 330 F F F
259562023-09-22T23:36:15.337Z 21 420-439 197 197 197 330 330 330 F F F
259572023-09-22T23:36:15.340Z 22 440-459 194 194 194 325 325 325 F F F
259582023-09-22T23:36:15.342Z 23 460-479 194 194 194 325 325 325 F F F
259592023-09-22T23:36:15.345Z 24 480-499 195 195 195 326 326 326 F F F
259602023-09-22T23:36:15.350Z 25 500-519 194 194 194 324 324 324 F F F
259612023-09-22T23:36:15.353Z 26 520-539 193 193 193 323 323 323 F F F
259622023-09-22T23:36:15.356Z 27 540-559 195 195 195 326 326 326 F F F
259632023-09-22T23:36:15.359Z 28 560-579 196 196 196 327 327 327 F F F
259642023-09-22T23:36:15.362Z 29 580-599 196 196 196 328 328 328 F F F
259652023-09-22T23:36:15.365ZMax gen: 197, Max flush: 330
259662023-09-22T23:36:15.368Z
259672023-09-22T23:36:15.370ZBegin loop 99
259682023-09-22T23:36:15.373ZDownstairs to restart: 2
259692023-09-22T23:36:15.376Z{"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.295936592Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
259702023-09-22T23:36:15.379Z{"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"ed48f294784d46ea7d4bb99336918b74358eca46\",\n git_commit_timestamp: \"2023-09-22T22:51:18.000000000Z\",\n git_branch: \"main\",\n rustc_semver: \"1.70.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"90c541806f23a127002de5b4038be731ba1458ca\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.296384987Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
259712023-09-22T23:36:15.382Z{"msg":"Upstairs <-> Downstairs Message Version: 4","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.296462795Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
259722023-09-22T23:36:15.385Z{"msg":"Crucible stats registered with UUID: 4bd5fb69-4b5b-459f-8106-0db93073f98c","v":0,"name":"crucible","level":30Crucible runtime is spawned
259732023-09-22T23:36:15.388Z,"time":"2023-09-22T23:36:15.296517752Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
259742023-09-22T23:36:15.390Z{"msg":"Crucible 4bd5fb69-4b5b-459f-8106-0db93073f98c has session id: 0bf40b2a-fd7d-46ad-a52c-9ba99dfeffd9","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.296648278Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975The guest has requested activation with gen:198
259752023-09-22T23:36:15.394Z}
259762023-09-22T23:36:15.397Z{"msg":"up_listen starts","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.296844191Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975,"task":"up_listen"}
259772023-09-22T23:36:15.400Z{"msg":"Wait for all three downstairs to come online","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.296967216Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
259782023-09-22T23:36:15.403Z{"msg":"Flush timeout: 0.5","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.297056956Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
259792023-09-22T23:36:15.406Z{"msg":"[2] connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.297133908Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975,"looper":"2"}
259802023-09-22T23:36:15.411Z{"msg":"[1] connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.297221104Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975,"looper":"1"}
259812023-09-22T23:36:15.414Z{"msg":"[1] 4bd5fb69-4b5b-459f-8106-0db93073f98c looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.297334452Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975,"looper":"1"}
259822023-09-22T23:36:15.418Z{"msg":"[0] connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.297432815Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975,"looper":"0"}
259832023-09-22T23:36:15.420Z{"msg":"[0] 4bd5fb69-4b5b-459f-8106-0db93073f98c looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.297547975Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975,"looper":"0"}
259842023-09-22T23:36:15.423Z{"msg":"4bd5fb69-4b5b-459f-8106-0db93073f98c active request set","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.297870231Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
259852023-09-22T23:36:15.427Z{"msg":"Set desired generation to :198","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.297927036Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
259862023-09-22T23:36:15.430Z{"msg":"[1] Proc runs for 127.0.0.1:8820 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.29802377Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
259872023-09-22T23:36:15.433Z{"msg":"[0] Proc runs for 127.0.0.1:8810 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.298370904Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
259882023-09-22T23:36:15.436Z{"msg":"[1] 4bd5fb69-4b5b-459f-8106-0db93073f98c (7b15b057-cb03-4a86-906f-825f4f5386a1) New New New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.298680692Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
259892023-09-22T23:36:15.439Z{"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.298765733Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
259902023-09-22T23:36:15.442Z{"msg":"[1] client is_active_req TRUE, promote! session 7b15b057-cb03-4a86-906f-825f4f5386a1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.29883721Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
259912023-09-22T23:36:15.445Z{"msg":"[0] 4bd5fb69-4b5b-459f-8106-0db93073f98c (7b15b057-cb03-4a86-906f-825f4f5386a1) New WaitActive New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.298910415Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
259922023-09-22T23:36:15.448Z{"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.29899822Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
259932023-09-22T23:36:15.452Z{"msg":"[0] client is_active_req TRUE, promote! session 7b15b057-cb03-4a86-906f-825f4f5386a1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.299071464Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
259942023-09-22T23:36:15.455Z{"msg":"[1] downstairs client at 127.0.0.1:8820 has UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.299195109Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
259952023-09-22T23:36:15.462Z{"msg":"[1] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.299273955Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
259962023-09-22T23:36:15.466Z{"msg":"4bd5fb69-4b5b-459f-8106-0db93073f98c WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.29934912Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
259972023-09-22T23:36:15.469Z{"msg":"[0] downstairs client at 127.0.0.1:8810 has UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.299428426Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
259982023-09-22T23:36:15.472Z{"msg":"[0] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.299509176Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
259992023-09-22T23:36:15.475Z{"msg":"4bd5fb69-4b5b-459f-8106-0db93073f98c WaitActive WaitActive New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.299583953Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260002023-09-22T23:36:15.478Z{"msg":"[1] 4bd5fb69-4b5b-459f-8106-0db93073f98c (7b15b057-cb03-4a86-906f-825f4f5386a1) WaitActive WaitActive New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.301576163Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260012023-09-22T23:36:15.481Z{"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.301718942Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260022023-09-22T23:36:15.484Z{"msg":"[1] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:36:15.30174576Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260032023-09-22T23:36:15.487Z{"msg":"[1] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.301784561Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260042023-09-22T23:36:15.490Z{"msg":"[1] 127.0.0.1:8820 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.301823542Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260052023-09-22T23:36:15.494Z{"msg":"4bd5fb69-4b5b-459f-8106-0db93073f98c WaitActive WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.301863312Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260062023-09-22T23:36:15.497Z{"msg":"Waiting for 2 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.30190201Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260072023-09-22T23:36:15.500Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.301939729Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260082023-09-22T23:36:15.503Z{"msg":"[0] 4bd5fb69-4b5b-459f-8106-0db93073f98c (7b15b057-cb03-4a86-906f-825f4f5386a1) WaitActive WaitQuorum New ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.301980424Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260092023-09-22T23:36:15.505Z{"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.302019201Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260102023-09-22T23:36:15.508Z{"msg":"[0] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:36:15.302057127Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260112023-09-22T23:36:15.511Z{"msg":"[0] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.302076699Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260122023-09-22T23:36:15.514Z{"msg":"[0] 127.0.0.1:8810 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.302094602Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260132023-09-22T23:36:15.517Z{"msg":"4bd5fb69-4b5b-459f-8106-0db93073f98c WaitQuorum WaitQuorum New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.302112585Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260142023-09-22T23:36:15.520Z{"msg":"Waiting for 1 more clients to be ready","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.302130614Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260152023-09-22T23:36:15.523Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:15.302148007Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260162023-09-22T23:36:16.300Z{"msg":"[2] 4bd5fb69-4b5b-459f-8106-0db93073f98c looper connected","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.297922535Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975,"looper":"2"}
260172023-09-22T23:36:16.303Z{"msg":"[2] Proc runs for 127.0.0.1:8830 in state New","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.298014927Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260182023-09-22T23:36:16.306Z{"msg":"[2] 4bd5fb69-4b5b-459f-8106-0db93073f98c (7b15b057-cb03-4a86-906f-825f4f5386a1) WaitQuorum WaitQuorum New ds_transition to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.299251932Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260192023-09-22T23:36:16.314Z{"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.299307987Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260202023-09-22T23:36:16.317Z{"msg":"[2] client is_active_req TRUE, promote! session 7b15b057-cb03-4a86-906f-825f4f5386a1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.299352348Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260212023-09-22T23:36:16.320Z{"msg":"[2] downstairs client at 127.0.0.1:8830 has UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.299925715Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260222023-09-22T23:36:16.324Z{"msg":"[2] Got region def RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008830, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.299985708Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260232023-09-22T23:36:16.327Z{"msg":"4bd5fb69-4b5b-459f-8106-0db93073f98c WaitQuorum WaitQuorum WaitActive","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.300028726Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260242023-09-22T23:36:16.330Z{"msg":"Checking if live repair is needed","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.302435536Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260252023-09-22T23:36:16.333Z{"msg":"[2] 4bd5fb69-4b5b-459f-8106-0db93073f98c (7b15b057-cb03-4a86-906f-825f4f5386a1) WaitQuorum WaitQuorum WaitActive ds_transition to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.302582839Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260262023-09-22T23:36:16.336Z{"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.302651595Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260272023-09-22T23:36:16.339Z{"msg":"[2] new RM replaced this: None","v":0,"name":"crucible","level":40,"time":"2023-09-22T23:36:16.302712188Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260282023-09-22T23:36:16.342Z{"msg":"[2] Starts reconcile loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.30276721Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260292023-09-22T23:36:16.345Z{"msg":"[2] 127.0.0.1:8830 task reports connection:true","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.302822942Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260302023-09-22T23:36:16.348Z{"msg":"4bd5fb69-4b5b-459f-8106-0db93073f98c WaitQuorum WaitQuorum WaitQuorum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.302879737Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260312023-09-22T23:36:16.350Z{"msg":"[0]R flush_numbers[0..12]: [330, 330, 324, 328, 291, 328, 328, 324, 330, 330, 330, 326]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.302918361Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260322023-09-22T23:36:16.353Z{"msg":"[0]R generation[0..12]: [197, 197, 194, 196, 180, 196, 196, 194, 197, 197, 197, 195]","v":0,"name":"The guest has finished waiting for activation with:198
260332023-09-22T23:36:16.356ZWait for a query_work_queue command to finish before sending IO
260342023-09-22T23:36:16.359Zcrucible","level":30,"time":"2023-09-22T23:36:16.303158261Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260352023-09-22T23:36:16.362Z{"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.303221474Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260362023-09-22T23:36:16.364Z{"msg":"[1]R flush_numbers[0..12]: [330, 330, 324, 328, 291, 328, 328, 324, 330, 330, 330, 326]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.303280369Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260372023-09-22T23:36:16.367Z{"msg":"[1]R generation[0..12]: [197, 197, 194, 196, 180, 196, 196, 194, 197, 197, 197, 195]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.303336756Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260382023-09-22T23:36:16.370Z{"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.303401811Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260392023-09-22T23:36:16.373Z{"msg":"[2]R flush_numbers[0..12]: [330, 330, 324, 328, 291, 328, 328, 324, 330, 330, 330, 326]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.303464965Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260402023-09-22T23:36:16.377Z{"msg":"[2]R generation[0..12]: [197, 197, 194, 196, 180, 196, 196, 194, 197, 197, 197, 195]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.303520526Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260412023-09-22T23:36:16.380Z{"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.3035775Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260422023-09-22T23:36:16.382Z{"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.303643267Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260432023-09-22T23:36:16.385Z{"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.303700667Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260442023-09-22T23:36:16.388Z{"msg":"Next flush: 331","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.303754592Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260452023-09-22T23:36:16.391Z{"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.303819126Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260462023-09-22T23:36:16.394Z{"msg":"No downstairs repair required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.303878076Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260472023-09-22T23:36:16.397Z{"msg":"No initial repair work was required","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.303938958Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260482023-09-22T23:36:16.400Z{"msg":"Set Downstairs and Upstairs active","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.303995415Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260492023-09-22T23:36:16.403Z{"msg":"4bd5fb69-4b5b-459f-8106-0db93073f98c is now active with session: 7b15b057-cb03-4a86-906f-825f4f5386a1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.304059677Z","hostname":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576
260502023-09-22T23:36:16.405Zip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260512023-09-22T23:36:16.409Z{"msg":"4bd5fb69-4b5b-459f-8106-0db93073f98c Set Active after no repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.304183695Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260522023-09-22T23:36:16.412Z{"msg":"Notify all downstairs, region set compare is done.","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.304240904Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260532023-09-22T23:36:16.415Z{"msg":"Set check for repair","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.304294477Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260542023-09-22T23:36:16.418Z{"msg":"[2] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.304348145Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260552023-09-22T23:36:16.421Z{"msg":"[2] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.304401853Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260562023-09-22T23:36:16.424Z{"msg":"[2] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.304454049Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260572023-09-22T23:36:16.427Z{"msg":"[0] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.304507544Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260582023-09-22T23:36:16.430Z{"msg":"[0] All repairs completed, exit","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.304561318Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260592023-09-22T23:36:16.433Z{"msg":"[0] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.304614314Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260602023-09-22T23:36:16.436Z{"msg":"[1] received reconcile message","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.304667427Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260612023-09-22T23:36:16.439ZLoading write count information from file "/tmp/test_repair_verify.data"
260622023-09-22T23:36:16.441Z{"msg":"[1] All repairs completed, exit","v":0,"name":"crucible","level":30Run Repair workload
260632023-09-22T23:36:16.444Z,"time":"2023-09-22T23:36:16.304725889Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260642023-09-22T23:36:16.447Z{"msg":"[1] Starts cmd_loop","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.304832992Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1975}
260652023-09-22T23:36:16.450Z01/30 Write block 207 len 40960 data: 22 26 20 18 18 14 13 13 14 13
260662023-09-22T23:36:16.452Z02/30 Read block 236 len 32768
260672023-09-22T23:36:16.455Z03/30 Write block 478 len 24576 data: 14 15 16 16 18 16
260682023-09-22T23:36:16.458Z04/30 Write block 502 len 36864 data: 17 14 14 14 14 12 9 16 17
260692023-09-22T23:36:16.462Z05/30 Write block 129 len 4096 data: 12
260702023-09-22T23:36:16.465Z06/30 Write block 52 len 36864 data: 12 10 10 11 13 16 17 17 19
260712023-09-22T23:36:16.469Z07/30 Read block 480 len 20480
260722023-09-22T23:36:16.471Z08/30 Write block 495 len 36864 data: 21 22 23 16 16 15 16 18 15
260732023-09-22T23:36:16.474Z09/30 Read block 16 len 36864
260742023-09-22T23:36:16.477Z10/30 Write block 291 len 28672 data: 16 24 20 18 18 18 14
260752023-09-22T23:36:16.480Z11/30 Write block 67 len 28672 data: 24 26 23 16 17 17 14
260762023-09-22T23:36:16.482Z12/30 Write block 110 len 24576 data: 25 22 22 21 20 16
260772023-09-22T23:36:16.485Z13/30 Write block 231 len 36864 data: 8 10 11 13 13 13 13 14 17
260782023-09-22T23:36:16.488Z14/30 Read block 334 len 8192
260792023-09-22T23:36:16.491Z15/30 Write block 89 len 32768 data: 14 13 15 15 15 14 16 16
260802023-09-22T23:36:16.493Z16/30 Flush
260812023-09-22T23:36:16.496Z17/30 Write block 189 len 36864 data: 12 14 15 14 15 16 17 16 14
260822023-09-22T23:36:16.499Z18/30 Read block 132 len 24576
260832023-09-22T23:36:16.501Z19/30 Flush
260842023-09-22T23:36:16.504Z20/30 Write block 162 len 24576 data: 18 15 13 12 10 10
260852023-09-22T23:36:16.507Z21/30 Read block 266 len 16384
260862023-09-22T23:36:16.510Z22/30 Write block 340 len 4096 data: 13
260872023-09-22T23:36:16.512Z23/30 Read block 45 len 4096
260882023-09-22T23:36:16.515Z24/30 Read block 405 len 36864
260892023-09-22T23:36:16.518Z25/30 Read block 110 len 36864
260902023-09-22T23:36:16.521Z26/30 Read block 368 len 20480
260912023-09-22T23:36:16.524Z27/30 Read block 494 len 16384
260922023-09-22T23:36:16.526Z28/30 Write block 47 len 16384 data: 15 13 14 12
260932023-09-22T23:36:16.529Z29/30 Write block 552 len 20480 data: 17 15 9 9 10
260942023-09-22T23:36:16.532Z30/30 Read block 348 len 40960
260952023-09-22T23:36:16.535Z----------------------------------------------------------------
260962023-09-22T23:36:16.537Z Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30
260972023-09-22T23:36:16.540ZGW_ID ACK DSID TYPE BLOCKS DS:0 DS:1 DS:2 REPLAY
260982023-09-22T23:36:16.543Z 1 Acked 1000 Write 10 Done Done Done false
260992023-09-22T23:36:16.546Z 2 Acked 1001 Read 8 Done Done Done false
261002023-09-22T23:36:16.549Z 3 Acked 1002 Write 6 Done Done Done false
261012023-09-22T23:36:16.551Z 4 Acked 1003 Write 9 Done Done Done false
261022023-09-22T23:36:16.554Z 5 Acked 1004 Write 1 Done Done Done false
261032023-09-22T23:36:16.557Z 6 Acked 1005 Write 9 Done Done Done false
261042023-09-22T23:36:16.560Z 7 Acked 1006 Read 5 Done Done Done false
261052023-09-22T23:36:16.563Z 8 Acked 1007 Write 9 Done Done Done false
261062023-09-22T23:36:16.566Z 9 Acked 1008 Read 9 Done Done Done false
261072023-09-22T23:36:16.569Z 10 Acked 1009 Write 7 Done Done Done false
261082023-09-22T23:36:16.572Z 11 Acked 1010 Write 7 Done Done Done false
261092023-09-22T23:36:16.575Z 12 Acked 1011 Write 6 Done Done Done false
261102023-09-22T23:36:16.578Z 13 Acked 1012 Write 9 Done Done Done false
261112023-09-22T23:36:16.601Z 14 Acked 1013 Read 2 Done Done Done false
261122023-09-22T23:36:16.604Z 15 Acked 1014 Write 8 Done Done Sent false
261132023-09-22T23:36:16.607Z 16 Acked 1015 Flush 0 Done Done Sent false
261142023-09-22T23:36:16.610Z 17 Acked 1016 Write 9 Done Done Sent false
261152023-09-22T23:36:16.613Z 18 Acked 1017 Read 6 Done Done Sent false
261162023-09-22T23:36:16.616Z 19 Acked 1018 Flush 0 Done Done Sent false
261172023-09-22T23:36:16.619Z 20 Acked 1019 Write 6 Done Done Sent false
261182023-09-22T23:36:16.622Z 21 Acked 1020 Read 4 Done Done Sent false
261192023-09-22T23:36:16.626Z 22 Acked 1021 Write 1 Done Done Sent false
261202023-09-22T23:36:16.629Z 23 Acked 1022 Read 1 Done Done Sent false
261212023-09-22T23:36:16.631Z 24 Acked 1023 Read 9 Done Done Sent false
261222023-09-22T23:36:16.634Z 25 Acked 1024 Read 9 Done Done Sent false
261232023-09-22T23:36:16.637Z 26 Acked 1025 Read 5 Done Done Sent false
261242023-09-22T23:36:16.640Z 27 Acked 1026 Read 4 Done Done Sent false
261252023-09-22T23:36:16.643Z 28 Acked 1027 Write 4 Done Done Sent false
261262023-09-22T23:36:16.646Z 29 Acked 1028 Write 5 Done Done Sent false
261272023-09-22T23:36:16.649Z 30 Acked 1029 Read 10 Sent Done Sent false
261282023-09-22T23:36:16.651Z STATES DS:0 DS:1 DS:2 TOTAL
261292023-09-22T23:36:16.654Z New 0 0 0 0
261302023-09-22T23:36:16.657Z Sent 1 0 16 17
261312023-09-22T23:36:16.660Z Done 29 30 14 73
261322023-09-22T23:36:16.663Z Skipped 0 0 0 0
261332023-09-22T23:36:16.666Z Error 0 0 0 0
261342023-09-22T23:36:16.668ZLast Flush: 1018 1018 0
261352023-09-22T23:36:16.671ZDownstairs last five completed:
261362023-09-22T23:36:16.674ZUpstairs last five completed: 30 29 28 27 26 25
261372023-09-22T23:36:16.677ZWrote out file "/tmp/test_repair_verify.data"
261382023-09-22T23:36:16.680Z
261392023-09-22T23:36:16.683ZCurrent downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830
261402023-09-22T23:36:16.686Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.454246293Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1976}
261412023-09-22T23:36:16.689Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.454585608Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1976}
261422023-09-22T23:36:16.691Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.454609162Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1976}
261432023-09-22T23:36:16.694Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.479877424Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1976}
261442023-09-22T23:36:16.697Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.479911216Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1976}
261452023-09-22T23:36:16.700Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.479930188Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1976}
261462023-09-22T23:36:16.703Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.496073261Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1976}
261472023-09-22T23:36:16.706Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.496132941Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1976}
261482023-09-22T23:36:16.709Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:16.496164584Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1976}
261492023-09-22T23:36:16.711ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
261502023-09-22T23:36:16.714Z 0 000-019 197 197 197 330 330 330 F F F
261512023-09-22T23:36:16.718Z 1 020-039 197 197 197 330 330 330 F F F
261522023-09-22T23:36:16.721Z 2 040-059 198 198 194 331 331 324 T T T <---
261532023-09-22T23:36:16.724Z 3 060-079 198 198 196 331 331 328 F F T <---
261542023-09-22T23:36:16.727Z 4 080-099 198 198 180 331 331 291 F F F <---
261552023-09-22T23:36:16.729Z 5 100-119 198 198 196 331 331 328 F F T <---
261562023-09-22T23:36:16.733Z 6 120-139 198 198 196 331 331 328 F F T <---
261572023-09-22T23:36:16.735Z 7 140-159 194 194 194 324 324 324 F F F
261582023-09-22T23:36:16.738Z 8 160-179 197 197 197 330 330 330 T T F <---
261592023-09-22T23:36:16.741Z 9 180-199 198 198 197 332 332 330 F F F <---
261602023-09-22T23:36:16.744Z 10 200-219 198 198 197 331 331 330 F F T <---
261612023-09-22T23:36:16.747Z 11 220-239 198 198 195 331 331 326 F F T <---
261622023-09-22T23:36:16.750Z 12 240-259 195 195 195 326 326 326 F F F
261632023-09-22T23:36:16.753Z 13 260-279 192 192 192 321 321 321 F F F
261642023-09-22T23:36:16.755Z 14 280-299 198 198 184 331 331 305 F F T <---
261652023-09-22T23:36:16.758Z 15 300-319 192 192 192 320 320 320 F F F
261662023-09-22T23:36:16.761Z 16 320-339 194 194 194 324 324 324 F F F
261672023-09-22T23:36:16.764Z 17 340-359 194 194 194 324 324 324 T T F <---
261682023-09-22T23:36:16.767ZError: Difference in extent metadata found!
261692023-09-22T23:36:16.770Z 18 360-379 196 196 196 328 328 328 F F F
261702023-09-22T23:36:16.773Z 19 380-399 188 188 188 315 315 315 F F F
261712023-09-22T23:36:16.776Z 20 400-419 197 197 197 330 330 330 F F F
261722023-09-22T23:36:16.780Z 21 420-439 197 197 197 330 330 330 F F F
261732023-09-22T23:36:16.784Z 22 440-459 194 194 194 325 325 325 F F F
261742023-09-22T23:36:16.787Z 23 460-479 198 198 194 331 331 325 F F T <---
261752023-09-22T23:36:16.790Z 24 480-499 198 198 195 331 331 326 F F T <---
261762023-09-22T23:36:16.793Z 25 500-519 198 198 194 331 331 324 F F T <---
261772023-09-22T23:36:16.796Z 26 520-539 193 193 193 323 323 323 F F F
261782023-09-22T23:36:16.798Z 27 540-559 195 195 195 326 326 326 T T F <---
261792023-09-22T23:36:16.801Z 28 560-579 196 196 196 327 327 327 F F F
261802023-09-22T23:36:16.804Z 29 580-599 196 196 196 328 328 328 F F F
261812023-09-22T23:36:16.807ZMax gen: 198, Max flush: 332
261822023-09-22T23:36:16.810ZA Difference in extent metadata is expected here
261832023-09-22T23:36:16.813ZOn loop 99
261842023-09-22T23:36:16.816Z
261852023-09-22T23:36:16.819ZVerifying data now
261862023-09-22T23:36:23.147ZNo change after verify
261872023-09-22T23:36:23.151ZLoop: 99 Downstairs dump after verify (and repair):
261882023-09-22T23:36:23.166Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:23.164261244Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1981}
261892023-09-22T23:36:23.169Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:23.164685093Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1981}
261902023-09-22T23:36:23.171Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:23.164710344Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1981}
261912023-09-22T23:36:23.185Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:23.183591452Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1981}
261922023-09-22T23:36:23.187Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:23.183648562Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1981}
261932023-09-22T23:36:23.190Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:23.183669789Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1981}
261942023-09-22T23:36:23.196Z{"msg":"current number of open files limit 65536 is already the maximum","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:23.194805564Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1981}
261952023-09-22T23:36:23.198Z{"msg":"Database read version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:23.194857387Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1981}
261962023-09-22T23:36:23.201Z{"msg":"Database write version 1","v":0,"name":"crucible","level":30,"time":"2023-09-22T23:36:23.194919769Z","hostname":"ip-10-150-1-177.us-west-2.compute.internal","pid":1981}
261972023-09-22T23:36:23.207ZEXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF
261982023-09-22T23:36:23.210Z 0 000-019 197 197 197 330 330 330 F F F
261992023-09-22T23:36:23.213Z 1 020-039 197 197 197 330 330 330 F F F
262002023-09-22T23:36:23.216Z 2 040-059 199 199 199 333 333 333 F F F
262012023-09-22T23:36:23.219Z 3 060-079 198 198 198 331 331 331 F F F
262022023-09-22T23:36:23.222Z 4 080-099 198 198 198 331 331 331 F F F
262032023-09-22T23:36:23.224Z 5 100-119 198 198 198 331 331 331 F F F
262042023-09-22T23:36:23.228Z 6 120-139 198 198 198 331 331 331 F F F
262052023-09-22T23:36:23.230Z 7 140-159 194 194 194 324 324 324 F F F
262062023-09-22T23:36:23.233Z 8 160-179 199 199 199 333 333 333 F F F
262072023-09-22T23:36:23.236Z 9 180-199 198 198 198 332 332 332 F F F
262082023-09-22T23:36:23.239Z 10 200-219 198 198 198 331 331 331 F F F
262092023-09-22T23:36:23.242Z 11 220-239 198 198 198 331 331 331 F F F
262102023-09-22T23:36:23.244Z 12 240-259 195 195 195 326 326 326 F F F
262112023-09-22T23:36:23.247Z 13 260-279 192 192 192 321 321 321 F F F
262122023-09-22T23:36:23.250Z 14 280-299 198 198 198 331 331 331 F F F
262132023-09-22T23:36:23.253Z 15 300-319 192 192 192 320 320 320 F F F
262142023-09-22T23:36:23.256Z 16 320-339 194 194 194 324 324 324 F F F
262152023-09-22T23:36:23.259Z 17 340-359 199 199 199 333 333 333 F F F
262162023-09-22T23:36:23.264Z 18 360-379 196 196 196 328 328 328 F F F
262172023-09-22T23:36:23.266Z 19 380-399 188 188 188 315 315 315 F F F
262182023-09-22T23:36:23.269Z 20 400-419 197 197 197 330 330 330 F F F
262192023-09-22T23:36:23.272Z 21 420-439 197 197 197 330 330 330 F F F
262202023-09-22T23:36:23.275Z 22 440-459 194 194 194 325 325 325 F F F
262212023-09-22T23:36:23.278Z 23 460-479 198 198 198 331 331 331 F F F
262222023-09-22T23:36:23.281Z 24 480-499 198 198 198 331 331 331 F F F
262232023-09-22T23:36:23.283Z 25 500-519 198 198 198 331 331 331 F F F
262242023-09-22T23:36:23.286Z 26 520-539 193 193 193 323 323 323 F F F
262252023-09-22T23:36:23.289Z 27 540-559 199 199 199 333 333 333 F F F
262262023-09-22T23:36:23.292Z 28 560-579 196 196 196 327 327 327 F F F
262272023-09-22T23:36:23.295Z 29 580-599 196 196 196 328 328 328 F F F
262282023-09-22T23:36:23.297ZMax gen: 199, Max flush: 333
262292023-09-22T23:36:23.300Z12:57 Test duration
262302023-09-22T23:36:23.303ZTest completed
262312023-09-22T23:36:23.306Z
262322023-09-22T23:36:23.309Zreal 12:55.800567603
262332023-09-22T23:36:23.312Zuser 3:35.767288464
262342023-09-22T23:36:23.314Zsys 37.143563460
262352023-09-22T23:36:23.318Ztrap 0.189606736
262362023-09-22T23:36:23.320Ztflt 0.096285782
262372023-09-22T23:36:23.323Zdflt 0.173213983
262382023-09-22T23:36:23.326Zkflt 0.001659231
262392023-09-22T23:36:23.329Zlock 8:57:05.177221173
262402023-09-22T23:36:23.332Zslp 1:01:06.316480225
262412023-09-22T23:36:23.335Zlat 11.152516045
262422023-09-22T23:36:23.338Zstop 0.066705651
262432023-09-22T23:36:23.341Zprocess exited: duration 788436 ms, exit code 0
 
262442023-09-22T23:36:23.354Zfound 4 output files
262452023-09-22T23:36:23.357Zuploading: /tmp/test_repair_ds8810.txt (86257 bytes)
262462023-09-22T23:36:24.368Zuploaded: /tmp/test_repair_ds8810.txt
262472023-09-22T23:36:24.371Zuploading: /tmp/test_repair_ds8820.txt (29927 bytes)
262482023-09-22T23:36:25.382Zuploaded: /tmp/test_repair_ds8820.txt
262492023-09-22T23:36:25.385Zuploading: /tmp/test_repair_ds8830.txt (23895 bytes)
262502023-09-22T23:36:26.394Zuploaded: /tmp/test_repair_ds8830.txt
262512023-09-22T23:36:26.401Zuploading: /tmp/test_repair_out.txt (279116 bytes)
262522023-09-22T23:36:27.415Zuploaded: /tmp/test_repair_out.txt